# dfir_ntfs: an NTFS parser for digital forensics & incident response
# (c) <NAME>
#
# This module implements an interface to work with the $LogFile journal.

import struct
import io
import shutil
import binascii

UPDATE_SEQUENCE_STRIDE = 512
PAGE_SIZE = 4096  # This is the size of a single memory page.

CLUSTER_SIZE_DEFAULT = 4096
FILE_RECORD_SEGMENT_SIZE_DEFAULT = 1024

# Values for the restart area:
LFS_NO_CLIENT = 0xFFFF  # No index in the array of client records.

# Flags for the restart area:
RESTART_SINGLE_PAGE_IO = 0x1  # Log entries are written one by one.
RESTART_UNKNOWN_NAME_2 = 0x2  # The new restart area has been written to the log file.

# Flags for the log record page:
LOG_PAGE_LOG_RECORD_END = 0x1  # This page contains the end of a log record.
LOG_PAGE_UNKNOWN_2 = 0x2  # This page contains a client restart area (set when the log file version is 2.0 or higher).

# Flags for the log record header:
LOG_RECORD_MULTI_PAGE = 0x1  # This record was written to multiple pages.
LOG_RECORD_UNKNOWN_NAME_2 = 0x2  # This record contains no redo buffer.
LOG_RECORD_UNKNOWN_NAME_4 = 0x4  # This record contains no undo buffer.

# Values for the log record types:
LfsClientRecord = 0x1
LfsClientRestart = 0x2

# Operations for the NTFS client:
Noop = 0x00
CompensationLogRecord = 0x01
InitializeFileRecordSegment = 0x02  # [1]
DeallocateFileRecordSegment = 0x03  # [1]
WriteEndOfFileRecordSegment = 0x04  # [1]
CreateAttribute = 0x05  # [1]
DeleteAttribute = 0x06  # [1]
UpdateResidentValue = 0x07  # [1]
UpdateNonresidentValue = 0x08  # [2]
UpdateMappingPairs = 0x09  # [1]
DeleteDirtyClusters = 0x0A
SetNewAttributeSizes = 0x0B  # [1]
AddIndexEntryRoot = 0x0C  # [1]
DeleteIndexEntryRoot = 0x0D  # [1]
AddIndexEntryAllocation = 0x0E  # [2]
DeleteIndexEntryAllocation = 0x0F  # [2]
WriteEndOfIndexBuffer = 0x10  # [2]
SetIndexEntryVcnRoot = 0x11  # [1]
SetIndexEntryVcnAllocation = 0x12  # [2]
UpdateFileNameRoot = 0x13  # [1]
UpdateFileNameAllocation = 0x14  # [2]
SetBitsInNonresidentBitMap = 0x15  # [2]
ClearBitsInNonresidentBitMap = 0x16  # [2]
HotFix = 0x17
EndTopLevelAction = 0x18
PrepareTransaction = 0x19
CommitTransaction = 0x1A
ForgetTransaction = 0x1B
OpenNonresidentAttribute = 0x1C
OpenAttributeTableDump = 0x1D
AttributeNamesDump = 0x1E
DirtyPageTableDump = 0x1F
TransactionTableDump = 0x20
UpdateRecordDataRoot = 0x21  # [1]
UpdateRecordDataAllocation = 0x22  # [2]
UpdateRelativeDataIndex = 0x23  # [1]
UpdateRelativeDataAllocation = 0x24  # [2]
ZeroEndOfFileRecord = 0x25  # [1]

# [1]: this is an update to the resident ($MFT) data.

LOGGED_RESIDENT_UPDATES = [InitializeFileRecordSegment, DeallocateFileRecordSegment, WriteEndOfFileRecordSegment,
                           CreateAttribute, DeleteAttribute, UpdateResidentValue, UpdateMappingPairs,
                           SetNewAttributeSizes, AddIndexEntryRoot, DeleteIndexEntryRoot, SetIndexEntryVcnRoot,
                           UpdateFileNameRoot, UpdateRecordDataRoot, UpdateRelativeDataIndex, ZeroEndOfFileRecord]

# [2]: this is an update to the nonresident data (which is referenced by an entry in the open attribute table).

LOGGED_NONRESIDENT_UPDATES = [UpdateNonresidentValue, AddIndexEntryAllocation, DeleteIndexEntryAllocation,
                              WriteEndOfIndexBuffer, SetIndexEntryVcnAllocation, UpdateFileNameAllocation,
                              SetBitsInNonresidentBitMap, ClearBitsInNonresidentBitMap, UpdateRecordDataAllocation,
                              UpdateRelativeDataAllocation]

NTFSOperations = {
    Noop: 'Noop',
    CompensationLogRecord: 'CompensationLogRecord',
    InitializeFileRecordSegment: 'InitializeFileRecordSegment',
    DeallocateFileRecordSegment: 'DeallocateFileRecordSegment',
    WriteEndOfFileRecordSegment: 'WriteEndOfFileRecordSegment',
    CreateAttribute: 'CreateAttribute',
    DeleteAttribute: 'DeleteAttribute',
    UpdateResidentValue: 'UpdateResidentValue',
    UpdateNonresidentValue: 'UpdateNonresidentValue',
    UpdateMappingPairs: 'UpdateMappingPairs',
    DeleteDirtyClusters: 'DeleteDirtyClusters',
    SetNewAttributeSizes: 'SetNewAttributeSizes',
    AddIndexEntryRoot: 'AddIndexEntryRoot',
    DeleteIndexEntryRoot: 'DeleteIndexEntryRoot',
    AddIndexEntryAllocation: 'AddIndexEntryAllocation',
    DeleteIndexEntryAllocation: 'DeleteIndexEntryAllocation',
    WriteEndOfIndexBuffer: 'WriteEndOfIndexBuffer',
    SetIndexEntryVcnRoot: 'SetIndexEntryVcnRoot',
    SetIndexEntryVcnAllocation: 'SetIndexEntryVcnAllocation',
    UpdateFileNameRoot: 'UpdateFileNameRoot',
    UpdateFileNameAllocation: 'UpdateFileNameAllocation',
    SetBitsInNonresidentBitMap: 'SetBitsInNonresidentBitMap',
    ClearBitsInNonresidentBitMap: 'ClearBitsInNonresidentBitMap',
    HotFix: 'HotFix',
    EndTopLevelAction: 'EndTopLevelAction',
    PrepareTransaction: 'PrepareTransaction',
    CommitTransaction: 'CommitTransaction',
    ForgetTransaction: 'ForgetTransaction',
    OpenNonresidentAttribute: 'OpenNonresidentAttribute',
    OpenAttributeTableDump: 'OpenAttributeTableDump',
    AttributeNamesDump: 'AttributeNamesDump',
    DirtyPageTableDump: 'DirtyPageTableDump',
    TransactionTableDump: 'TransactionTableDump',
    UpdateRecordDataRoot: 'UpdateRecordDataRoot',
    UpdateRecordDataAllocation: 'UpdateRecordDataAllocation',
    UpdateRelativeDataIndex: 'UpdateRelativeDataIndex',
    UpdateRelativeDataAllocation: 'UpdateRelativeDataAllocation',
    ZeroEndOfFileRecord: 'ZeroEndOfFileRecord'
}


def ResolveNTFSOperation(NTFSOperationCode):
    """Convert an NTFS operation code to a string."""

    if NTFSOperationCode in NTFSOperations.keys():
        return NTFSOperations[NTFSOperationCode]

    return hex(NTFSOperationCode)


def UnprotectSectors(Buffer):
    """Apply an update sequence array (USA) to multiple sectors (as a bytearray object) and return the (resulting buffer, end_of_usa_offset) tuple."""

    if len(Buffer) < 2 * UPDATE_SEQUENCE_STRIDE or len(Buffer) % UPDATE_SEQUENCE_STRIDE != 0:
        raise UpdateSequenceArrayException('Invalid input (not two or more sectors)')

    usa_offset, usa_size = struct.unpack('<HH', Buffer[4: 8])

    if usa_offset < 8 or usa_offset > UPDATE_SEQUENCE_STRIDE - 6:
        raise UpdateSequenceArrayException('Invalid update sequence array offset: {}'.format(usa_offset))

    if usa_size < 2 or (usa_size - 1) * UPDATE_SEQUENCE_STRIDE != len(Buffer):
        raise UpdateSequenceArrayException('Invalid update sequence array size: {}'.format(usa_size))

    if usa_offset + usa_size * 2 >= len(Buffer):
        raise UpdateSequenceArrayException(
            'Invalid update sequence array offset and size: {}, {}'.format(usa_offset, usa_size))

    if Buffer[
       usa_offset + 4: usa_offset + 8] == b'CRC4':  # Check if integrity of data in this block is protected using the CRC32 value (for DAX volumes).
        crc32_recorded_raw = Buffer[usa_offset: usa_offset + 4]
        crc32_recorded = struct.unpack('<L', crc32_recorded_raw)[0]

        buffer_modified = Buffer[:]
        buffer_modified[usa_offset: usa_offset + usa_size * 2] = b'\x00' * (usa_size * 2)

        crc32_calculated = binascii.crc32(buffer_modified)

        if crc32_recorded == crc32_calculated:
            end_of_usa_offset = usa_offset + usa_size * 2
            return (Buffer, end_of_usa_offset)

    # If the checksum value is invalid, treat this block as a usual one.

    sequence_number_in_usa_bytes = Buffer[usa_offset: usa_offset + 2]

    i = 1  # Skip the first element (sequence_number_in_usa_bytes).
    while i < usa_size:
        offset_in_usa = i * 2
        update_bytes = Buffer[usa_offset + offset_in_usa: usa_offset + offset_in_usa + 2]

        offset_in_buf = i * UPDATE_SEQUENCE_STRIDE - 2
        sequence_number_in_sector_bytes = Buffer[offset_in_buf: offset_in_buf + 2]

        if sequence_number_in_usa_bytes != sequence_number_in_sector_bytes:
            raise UpdateSequenceArrayException(
                'Invalid sequence number in the buffer, relative offset: {}'.format(offset_in_buf))

        Buffer[offset_in_buf] = update_bytes[0]
        Buffer[offset_in_buf + 1] = update_bytes[1]

        i += 1

    end_of_usa_offset = usa_offset + usa_size * 2
    return (Buffer, end_of_usa_offset)


class LogFileException(Exception):
    """This is a top-level exception for this module."""

    def __init__(self, value):
        self._value = value

    def __str__(self):
        return repr(self._value)


class UpdateSequenceArrayException(LogFileException):
    """This exception is raised when something is wrong with an update sequence array."""

    pass


class RestartPageException(LogFileException):
    """This exception is raised when something is wrong with a restart page."""

    pass


class EmptyLogFileException(RestartPageException):
    """This exception is raised when the log file is empty."""

    pass


class RestartAreaException(LogFileException):
    """This exception is raised when something is wrong with a restart area."""

    pass


class ClientRecordException(LogFileException):
    """This exception is raised when something is wrong with a client record."""

    pass


class LogRecordPageException(LogFileException):
    """This exception is raised when something is wrong with a log record page."""

    pass


class ClientException(LogFileException):
    """This exception is raised when something is wrong with client data."""

    pass


class RestartPage(object):
    """This class is used to work with a restart page."""

    buf = None
    """Data of this restart page with updates from an update sequence array (USA) applied."""

    def __init__(self, restart_page_raw):
        self.buf, __ = UnprotectSectors(bytearray(restart_page_raw))

        signature = self.buf[: 4]
        if signature not in [b'RSTR', b'CHKD', b'BAAD']:
            raise RestartPageException('Invalid signature: {}'.format(signature))

        major_version = self.get_major_version()
        minor_version = self.get_minor_version()

        if (major_version, minor_version) not in [(1, 1), (2, 0), (3, 0)]:
            version_str = '{}.{}'.format(major_version, minor_version)
            raise NotImplementedError('Version not supported: {}'.format(version_str))

        system_page_size = self.get_system_page_size()
        if system_page_size < 1024 or system_page_size % 512 != 0:
            raise RestartPageException('Invalid system page size: {}'.format(system_page_size))

        if system_page_size != PAGE_SIZE:
            raise NotImplementedError('The following system page size is not supported: {}'.format(system_page_size))

        log_page_size = self.get_log_page_size()
        if log_page_size < 1024 or log_page_size % 512 != 0:
            raise RestartPageException('Invalid log page size: {}'.format(log_page_size))

        if log_page_size != PAGE_SIZE:
            raise NotImplementedError('The following log page size is not supported: {}'.format(log_page_size))

        restart_offset = self.get_restart_offset()
        if restart_offset == 0 or restart_offset % 8 != 0 or restart_offset >= len(
                self.buf) or restart_offset >= system_page_size:
            raise RestartPageException('Invalid restart offset: {}'.format(restart_offset))

    def get_chkdsk_lsn(self):
        """Get and return the last log sequence number (LSN) found during the chkdsk scan."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_system_page_size(self):
        """Get and return the system page size. This is the size of a restart page."""

        return struct.unpack('<L', self.buf[16: 20])[0]

    def get_log_page_size(self):
        """Get and return the log page size. This is the size of a log record page."""

        return struct.unpack('<L', self.buf[20: 24])[0]

    def get_restart_offset(self):
        """Get and return the restart area offset (relative from the start of this structure)."""

        return struct.unpack('<H', self.buf[24: 26])[0]

    def get_restart_area(self):
        """Get, parse and return the restart area (as a RestartArea object)."""

        return RestartArea(self.buf[self.get_restart_offset():], self.get_system_page_size())

    def get_major_version(self):
        """Get and return the major version of the log file."""

        return struct.unpack('<H', self.buf[28: 30])[0]

    def get_minor_version(self):
        """Get and return the minor version of the log file."""

        return struct.unpack('<H', self.buf[26: 28])[0]

    def __str__(self):
        return 'RestartPage, version: {}.{}, system page size: {}, log page size: {}'.format(self.get_major_version(),
                                                                                             self.get_minor_version(),
                                                                                             self.get_system_page_size(),
                                                                                             self.get_log_page_size())


class RestartArea(object):
    """This class is used to work with a restart area."""

    buf = None
    """Data of this restart area."""

    def __init__(self, restart_area_raw, system_page_size):
        self.buf = restart_area_raw

        client_array_offset = self.get_client_array_offset()
        if client_array_offset == 0 or client_array_offset % 8 != 0 or client_array_offset >= len(
                self.buf) or client_array_offset >= system_page_size:
            raise RestartAreaException('Invalid client array offset: {}'.format(client_array_offset))

        log_clients = self.get_log_clients()
        client_free_index = self.get_client_free_list()
        client_in_use_index = self.get_client_in_use_list()

        if client_free_index != LFS_NO_CLIENT and client_free_index >= log_clients:
            raise RestartAreaException('Invalid client index (free): {}'.format(client_free_index))

        if client_in_use_index != LFS_NO_CLIENT and client_in_use_index >= log_clients:
            raise RestartAreaException('Invalid client index (in-use): {}'.format(client_in_use_index))

    def get_current_lsn(self):
        """Get and return the current log sequence number (LSN)."""

        return struct.unpack('<Q', self.buf[0: 8])[0]

    def get_log_clients(self):
        """Get and return the number of log clients."""

        return struct.unpack('<H', self.buf[8: 10])[0]

    def get_client_free_list(self):
        """Get and return the index of the free list of client records."""

        return struct.unpack('<H', self.buf[10: 12])[0]

    def get_client_in_use_list(self):
        """Get and return the index of the in-use list of client records."""

        return struct.unpack('<H', self.buf[12: 14])[0]

    def get_flags(self):
        """Get and return the flags (as an integer)."""

        return struct.unpack('<H', self.buf[14: 16])[0]

    def get_sequence_number_bits(self):
        """Get and return the number of bits to use for the sequence number."""

        return struct.unpack('<L', self.buf[16: 20])[0]

    def get_restart_area_length(self):
        """Get and return the size of this restart area."""

        return struct.unpack('<H', self.buf[20: 22])[0]

    def get_client_array_offset(self):
        """Get and return the offset to the array of client records (relative from the start of this structure)."""

        return struct.unpack('<H', self.buf[22: 24])[0]

    def get_file_size(self):
        """Get and return the log file size."""

        return struct.unpack('<q', self.buf[24: 32])[0]

    def get_last_lsn_data_length(self):
        """Get and return the size of data from the last log sequence number (LSN), excluding the header."""

        return struct.unpack('<L', self.buf[32: 36])[0]

    def get_record_header_length(self):
        """Get and return the size of the log page header."""

        return struct.unpack('<H', self.buf[36: 38])[0]

    def get_log_page_data_offset(self):
        """Get and return the log page data offset."""

        return struct.unpack('<H', self.buf[38: 40])[0]

    def get_revision_number(self):
        """Get and return the revision number for this area."""

        return struct.unpack('<L', self.buf[40: 44])[0]

    def clients(self):
        """This method yields client records (ClientRecord objects)."""

        i = 0
        while i < self.get_log_clients():
            curr_pos = i * ClientRecord.sizeof
            buf = self.buf[curr_pos + self.get_client_array_offset():]

            yield ClientRecord(buf)

            i += 1

    def __str__(self):
        return 'RestartArea, clients: {} (free index: {}, in-use index: {}), flags: {}, current LSN: {}, sequence number bits: {}'.format(
            self.get_log_clients(), hex(self.get_client_free_list()), hex(self.get_client_in_use_list()),
            hex(self.get_flags()), self.get_current_lsn(), self.get_sequence_number_bits())


class ClientRecord(object):
    """This class is used to work with a client record."""

    buf = None
    """Data of this client record."""

    sizeof = 160
    """An allocated size of this record."""

    def __init__(self, client_record_raw):
        self.buf = client_record_raw

        client_name_length = self.get_client_name_length()
        if client_name_length > 128:  # Do not cross the 'sizeof' boundary.
            raise ClientRecordException('Invalid name length: {}'.format(client_name_length))

    def get_oldest_lsn(self):
        """Get and return the oldest log sequence number (LSN) required by this client."""

        return struct.unpack('<Q', self.buf[0: 8])[0]

    def get_client_restart_lsn(self):
        """Get and return the log sequence number (LSN) of the latest client restart area."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_prev_client(self):
        """Get and return the index of the previous client record."""

        return struct.unpack('<H', self.buf[16: 18])[0]

    def get_next_client(self):
        """Get and return the index of the next client record."""

        return struct.unpack('<H', self.buf[18: 20])[0]

    def get_sequence_number(self):
        """Get and return the sequence number."""

        return struct.unpack('<H', self.buf[20: 22])[0]

    def get_client_name_length(self):
        """Get and return the name length for this client."""

        return struct.unpack('<L', self.buf[28: 32])[0]

    def get_client_name(self):
        """Get, decode and return the name of this client."""

        name_raw = self.buf[32: 32 + self.get_client_name_length()]
        return name_raw.decode('utf-16le')

    def __str__(self):
        return 'ClientRecord, name: {}, oldest LSN: {}, client restart LSN: {}, sequence number: {}'.format(
            self.get_client_name(), self.get_oldest_lsn(), self.get_client_restart_lsn(), self.get_sequence_number())


class LogRecordPage(object):
    """This class is used to work with a log record page."""

    buf = None
    """Data of this log record page with updates from an update sequence array (USA) applied."""

    buf_protected = None
    """Data of this log record page without updates from an update sequence array (USA) applied."""

    end_of_usa_offset = None
    """Offset to the end of an update sequence array (USA) in the data."""

    data_offset = None
    """Offset to the start of the log data."""

    page_number = None
    """Number of this page in the log file."""

    def __init__(self, log_record_page_raw, log_page_number, data_offset):
        self.buf, self.end_of_usa_offset = UnprotectSectors(bytearray(log_record_page_raw))
        self.buf_protected = log_record_page_raw

        signature = self.buf[: 4]
        if signature not in [b'RCRD', b'CHKD', b'BAAD']:
            raise LogRecordPageException('Invalid signature: {}'.format(signature))

        self.page_number = log_page_number

        if data_offset < 8 or data_offset < self.end_of_usa_offset or data_offset % 8 != 0:
            raise NotImplementedError(
                'The following log data offset is invalid or not supported: {}'.format(data_offset))

        self.data_offset = data_offset

    def get_last_lsn(self):
        """Get and return the highest log sequence number (LSN) in this page."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_file_offset_11(self):
        """Get and return the log file offset for a tail copy. The log file version should be 1.1. This field is not valid for regular log record pages."""

        return struct.unpack('<q', self.buf[8: 16])[0]  # The same offset as above!

    def get_file_offset_2x(self):
        """Get and return the log file offset for a fast copy. The log file version should be 2.0 (or higher). This field is not valid for regular log record pages."""

        if self.end_of_usa_offset <= 60:
            return struct.unpack('<L', self.buf[60: 64])[0]

    def get_flags(self):
        """Get and return the flags (as an integer)."""

        return struct.unpack('<L', self.buf[16: 20])[0]

    def get_page_count(self):
        """Get and return the page count."""

        return struct.unpack('<H', self.buf[20: 22])[0]

    def get_page_position(self):
        """Get and return the page position."""

        return struct.unpack('<H', self.buf[22: 24])[0]

    def get_next_record_offset(self):
        """Get and return the offset of the free space in this page."""

        return struct.unpack('<H', self.buf[24: 26])[0]

    def get_last_end_lsn(self):
        """Get and return the log sequence number (LSN) for the last record in this page."""

        return struct.unpack('<Q', self.buf[32: 40])[0]

    def get_log_record_at_offset(self, offset, header_length=None):
        """Get, parse and return a log record header (LogRecordHeader) by its offset in this log record page."""

        if offset < self.data_offset:
            raise LogFileException('Invalid log data offset: {} < {}'.format(offset, self.data_offset))

        return LogRecordHeader(self.buf[offset:], header_length)

    def __str__(self):
        return 'LogRecordPage #{}, USA end: {}, last LSN: {} (or file offset 1.1/2.x: {}/{}), flags: {}, page count: {}, page position: {}, last end LSN: {}'.format(
            self.page_number, self.end_of_usa_offset, self.get_last_lsn(), self.get_file_offset_11(),
            self.get_file_offset_2x(), hex(self.get_flags()), self.get_page_count(), self.get_page_position(),
            self.get_last_end_lsn())


class LogRecordHeader(object):
    """This class is used to work with a log record header."""

    buf = None
    """Data of this log record (including client data)."""

    sizeof = 48
    """A size of this header (aligned to 8 bytes)."""

    def __init__(self, log_record_raw, header_length=None):
        self.buf = log_record_raw

        if header_length is not None and header_length < 48:
            raise NotImplementedError(
                'The following log record header length is not supported: {}'.format(header_length))

        if header_length is not None:
            self.sizeof = header_length

        if self.sizeof < 8 or self.sizeof % 8 != 0:
            raise LogFileException('Invalid log header length: {}'.format(self.sizeof))

        if len(self.buf) < self.sizeof:
            raise LogFileException('Truncated log record header')

    def get_this_lsn(self):
        """Get and return the log sequence number (LSN) for this record."""

        return struct.unpack('<Q', self.buf[0: 8])[0]

    def get_client_previous_lsn(self):
        """Get and return the previous log sequence number (LSN) for this client."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_client_undo_next_lsn(self):
        """Get and return the next undo log sequence number (LSN) for this client."""

        return struct.unpack('<Q', self.buf[16: 24])[0]

    def get_client_data_length(self):
        """Get and return the client data length for this record."""

        return struct.unpack('<L', self.buf[24: 28])[0]

    def get_client_id(self):
        """Get and return the client ID for this record."""

        return struct.unpack('<L', self.buf[28: 32])[0]

    def get_record_type(self):
        """Get and return the record type."""

        return struct.unpack('<L', self.buf[32: 36])[0]

    def get_transaction_id(self):
        """Get and return the transaction ID."""

        return struct.unpack('<L', self.buf[36: 40])[0]

    def get_flags(self):
        """Get and return the flags (as an integer)."""

        return struct.unpack('<H', self.buf[40: 42])[0]

    def __str__(self):
        return 'LogRecordHeader, this LSN: {}, client ID: {}, record type: {}, client data length: {}, transaction ID: {}, flags: {}'.format(
            self.get_this_lsn(), self.get_client_id(), self.get_record_type(), self.get_client_data_length(),
            self.get_transaction_id(), hex(self.get_flags()))


class LogFileParser(object):
    """This class is used to read and parse a $LogFile file."""

    file_object = None
    """A file object for a $LogFile file."""

    file_size = None
    """A size of this $LogFile file."""

    restart_page_1 = None
    """The first restart page (a RestartPage object)."""

    restart_page_2 = None
    """The second restart page (a RestartPage object)."""

    log_page_size = None
    """A size of a single log record page."""

    log_version = None
    """A tuple (major_version, minor_version) for this log file."""

    lsns = None
    """A sorted list of log sequence numbers (LSNs) seen in this file."""

    lsns_sorted = None
    """A dictionary with a sorted list of log sequence numbers (LSNs) grouped by client IDs (each client ID is a key in this dictionary)."""

    cluster_size = None
    """A number of bytes per cluster. (It will be adjusted later.)"""

    oat_version = None
    """A version of the open attribute table to use. (It will be adjusted later.)"""

    def __init__(self, file_object):
        self.file_object = file_object
        self.writable = False

        self.file_object.seek(0, 2)
        self.file_size = self.file_object.tell()

        self.file_object.seek(0)
        restart_page_1_raw = self.file_object.read(PAGE_SIZE)
        restart_page_2_raw = self.file_object.read(PAGE_SIZE)

        if len(restart_page_1_raw) != PAGE_SIZE or len(restart_page_2_raw) != PAGE_SIZE:
            raise LogFileException('Truncated log file')

        if restart_page_1_raw == restart_page_2_raw == b'\xFF' * PAGE_SIZE:
            raise EmptyLogFileException('Empty log file')

        try:
            self.restart_page_1 = RestartPage(restart_page_1_raw)
        except LogFileException:
            self.restart_page_1 = None

        try:
            self.restart_page_2 = RestartPage(restart_page_2_raw)
        except LogFileException:
            self.restart_page_2 = None

        if self.restart_page_1 is None and self.restart_page_2 is None:  # Both restart pages are corrupt.
            raise LogFileException('No valid restart pages found')

        log_page_size = None
        log_version = None

        if self.restart_page_1 is None:  # The first restart page is corrupt, use the second one.
            log_page_size = self.restart_page_2.get_log_page_size()
            log_version = (self.restart_page_2.get_major_version(), self.restart_page_2.get_minor_version())
            restart_area = self.restart_page_2.get_restart_area()

        if self.restart_page_2 is None:  # The second restart page is corrupt, use the first one.
            log_page_size = self.restart_page_1.get_log_page_size()
            log_version = (self.restart_page_1.get_major_version(), self.restart_page_1.get_minor_version())
            restart_area = self.restart_page_1.get_restart_area()

        if log_page_size is None:  # Both restart pages are valid.
            log_page_size = self.restart_page_1.get_log_page_size()
            restart_area = self.restart_page_1.get_restart_area()

            if log_page_size != self.restart_page_2.get_log_page_size():
                raise LogFileException('Invalid log page size in the second restart page')

            log_version = (self.restart_page_1.get_major_version(), self.restart_page_1.get_minor_version())
            if log_version != (self.restart_page_2.get_major_version(), self.restart_page_2.get_minor_version()):
                raise LogFileException('Invalid log version the second restart page')

        self.log_page_size = log_page_size
        self.log_version = log_version

        self.record_header_length = restart_area.get_record_header_length()
        if self.record_header_length < LogRecordHeader.sizeof:
            raise LogFileException(
                'Invalid record header length: {} < {}'.format(self.record_header_length, LogRecordHeader.sizeof))

        self.log_page_data_offset = restart_area.get_log_page_data_offset()

        self.sequence_number_bits = restart_area.get_sequence_number_bits()
        if self.sequence_number_bits >= 64 or self.sequence_number_bits < 3:
            raise LogFileException('Invalid count of sequence number bits: {}'.format(self.sequence_number_bits))

        self.cluster_size = CLUSTER_SIZE_DEFAULT  # Adjusted in the 'collect_lsns' method.
        self.oat_version = 1  # Adjusted in the 'collect_lsns' method.

    def lsn_to_offset(self, lsn):
        """Convert a log sequence number (LSN) to a file offset, return a (log_page_number, offset_in_page) tuple."""

        absolute_offset = ((lsn << self.sequence_number_bits) & 0xFFFFFFFFFFFFFFFF) >> (self.sequence_number_bits - 3)

        log_page_number = absolute_offset // self.log_page_size
        offset_in_page = absolute_offset % self.log_page_size

        return (log_page_number, offset_in_page)

    def lsn_to_seqnum(self, lsn):
        """Convert a log sequence number (LSN) to a sequence number and return it."""

        return lsn >> (64 - self.sequence_number_bits)

    def offset_to_lsn(self, log_page_number, offset_in_page, sequence_number):
        """Convert a log page number, an offset within that page, and a sequence number to a log sequence number (LSN) and return it."""

        absolute_offset = log_page_number * self.log_page_size + offset_in_page

        return (absolute_offset >> 3) | (sequence_number << (64 - self.sequence_number_bits))

    def get_log_record_page_by_number(self, log_page_number):
        """Get, parse and return a log record page (LogRecordPage) by its number (or None, if not found)."""

        self.file_object.seek(log_page_number * self.log_page_size)
        buf = self.file_object.read(self.log_page_size)

        if len(buf) != self.log_page_size:  # Truncated data.
            return

        try:
            log_page = LogRecordPage(buf, log_page_number, self.log_page_data_offset)
        except (LogFileException, NotImplementedError):
            log_page = None

        return log_page

    def get_wrapped_log_record_page_number(self):
        """Return the first log record page number to be used after the log file is wrapped."""

        if self.log_version[0] == 1 and self.log_version[1] == 1:
            start_page = 4  # 2 restart pages and 2 tail log record pages.
        elif self.log_version[0] >= 2:
            start_page = 34  # 2 restart pages and 32 fast pages.
        else:
            raise NotImplementedError('Unknown location of the "infinite" area')

        return start_page

    def collect_lsns(self):
        """Collect valid log sequence numbers (LSNs) found in this log file."""

        self.lsns = set()

        if self.restart_page_1 is not None:
            lsn = self.restart_page_1.get_chkdsk_lsn()
            self.lsns.add(lsn)

            lsn = self.restart_page_1.get_restart_area().get_current_lsn()
            self.lsns.add(lsn)

            for client in self.restart_page_1.get_restart_area().clients():
                lsn = client.get_oldest_lsn()
                self.lsns.add(lsn)

                lsn = client.get_client_restart_lsn()
                self.lsns.add(lsn)

        if self.restart_page_2 is not None:
            lsn = self.restart_page_2.get_chkdsk_lsn()
            self.lsns.add(lsn)

            lsn = self.restart_page_2.get_restart_area().get_current_lsn()
            self.lsns.add(lsn)

            for client in self.restart_page_2.get_restart_area().clients():
                lsn = client.get_oldest_lsn()
                self.lsns.add(lsn)

                lsn = client.get_client_restart_lsn()
                self.lsns.add(lsn)

        for log_record_page in self.log_record_pages(1):
            lsn = log_record_page.get_last_lsn()
            self.lsns.add(lsn)

            lsn = log_record_page.get_last_end_lsn()
            self.lsns.add(lsn)

        # Validate every LSN found, collect new LSNs.
        new_lsns = set()
        for lsn in self.lsns.copy():
            log_page_number, offset = self.lsn_to_offset(lsn)

            if log_page_number == 0:
                self.lsns.remove(lsn)
                continue

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                self.lsns.remove(lsn)
                continue

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset, self.record_header_length)
            except LogFileException:
                self.lsns.remove(lsn)
                continue

            if lsn != log_record_header.get_this_lsn():
                self.lsns.remove(lsn)

            new_lsn = log_record_header.get_client_previous_lsn()
            new_lsns.add(new_lsn)

            new_lsn = log_record_header.get_client_undo_next_lsn()
            new_lsns.add(new_lsn)

        # Validate these new LSNs.
        for lsn in new_lsns.copy():
            log_page_number, offset = self.lsn_to_offset(lsn)

            if log_page_number == 0:
                new_lsns.remove(lsn)
                continue

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                new_lsns.remove(lsn)
                continue

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset, self.record_header_length)
            except LogFileException:
                new_lsns.remove(lsn)
                continue

            if lsn != log_record_header.get_this_lsn():
                new_lsns.remove(lsn)

        # Extend the list of LSNs found using new data.
        self.lsns.update(new_lsns)

        # Now, walk through all LSNs found before and locate new LSNs.

        def find_next_lsn(current_lsn):
            """Given a current LSN, find, validate and return the next LSN. If there is none or something is invalid, return None."""

            log_page_number, offset_in_page = self.lsn_to_offset(current_lsn)
            if log_page_number == 0:
                return

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                return

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset_in_page, self.record_header_length)
            except LogFileException:
                return

            client_data_length = log_record_header.get_client_data_length()
            if client_data_length < 8 or client_data_length % 8 != 0:
                return

            seqnum_expected = self.lsn_to_seqnum(lsn)

            if client_data_length <= self.log_page_size - offset_in_page - self.record_header_length:  # Client data is in the current log record page only.
                # Try the current log record page.
                candidate_offset_in_page = offset_in_page + self.record_header_length + client_data_length
                candidate_lsn = self.offset_to_lsn(log_page_number, candidate_offset_in_page, seqnum_expected)

                try:
                    candidate_log_record_header = log_record_page.get_log_record_at_offset(candidate_offset_in_page,
                                                                                           self.record_header_length)
                except LogFileException:
                    pass
                else:
                    if candidate_log_record_header.get_this_lsn() == candidate_lsn:
                        return candidate_lsn

                # Try the next log record page.
                candidate_log_page_number = log_page_number + 1
                candidate_offset_in_page = self.log_page_data_offset
                candidate_lsn = self.offset_to_lsn(candidate_log_page_number, candidate_offset_in_page, seqnum_expected)

                # Handle the wrap.
                if candidate_log_page_number * self.log_page_size >= self.file_size:
                    candidate_log_page_number = self.get_wrapped_log_record_page_number()
                    seqnum_expected += 1
                    candidate_lsn = self.offset_to_lsn(candidate_log_page_number, candidate_offset_in_page,
                                                       seqnum_expected)

                candidate_log_record_page = self.get_log_record_page_by_number(candidate_log_page_number)
                if candidate_log_record_page is not None:
                    try:
                        candidate_log_record_header = candidate_log_record_page.get_log_record_at_offset(
                            candidate_offset_in_page, self.record_header_length)
                    except LogFileException:
                        pass
                    else:
                        if candidate_log_record_header.get_this_lsn() == candidate_lsn:
                            return candidate_lsn

            else:  # Client data is in two or more log record pages.
                client_data_length_left = client_data_length - (
                        self.log_page_size - offset_in_page - self.record_header_length)  # Client data length in the next pages (without the first one).

                pages_to_skip = 1 + client_data_length_left // (
                        self.log_page_size - self.log_page_data_offset)  # The number of pages to skip (including the first one with client data).

                candidate_log_page_number = log_page_number + pages_to_skip

                candidate_offset_in_page = self.log_page_data_offset + client_data_length_left - (pages_to_skip - 1) * (
                        self.log_page_size - self.log_page_data_offset)
                candidate_lsn = self.offset_to_lsn(candidate_log_page_number, candidate_offset_in_page, seqnum_expected)

                # Handle the wrap.
                if candidate_log_page_number * self.log_page_size >= self.file_size:
                    pages_left_in_file = (self.file_size // self.log_page_size) - (
                            log_page_number + 1)  # Pages left after the first one with client data.

                    candidate_log_page_number = self.get_wrapped_log_record_page_number() + (
                            pages_to_skip - 1) - pages_left_in_file
                    seqnum_expected += 1
                    candidate_lsn = self.offset_to_lsn(candidate_log_page_number, candidate_offset_in_page,
                                                       seqnum_expected)

                candidate_log_record_page = self.get_log_record_page_by_number(candidate_log_page_number)
                if candidate_log_record_page is not None:
                    try:
                        candidate_log_record_header = candidate_log_record_page.get_log_record_at_offset(
                            candidate_offset_in_page, self.record_header_length)
                    except LogFileException:
                        pass
                    else:
                        if candidate_log_record_header.get_this_lsn() == candidate_lsn:
                            return candidate_lsn

                # Try the next log record page.
                candidate_log_page_number += 1
                candidate_offset_in_page = self.log_page_data_offset
                candidate_lsn = self.offset_to_lsn(candidate_log_page_number, candidate_offset_in_page, seqnum_expected)

                candidate_log_record_page = self.get_log_record_page_by_number(candidate_log_page_number)
                if candidate_log_record_page is not None:
                    try:
                        candidate_log_record_header = candidate_log_record_page.get_log_record_at_offset(
                            candidate_offset_in_page, self.record_header_length)
                    except LogFileException:
                        pass
                    else:
                        if candidate_log_record_header.get_this_lsn() == candidate_lsn:
                            return candidate_lsn

        def find_previous_lsn(current_lsn):
            """Give a current LSN, find, validate and return the previous LSN for the same client. If there is none or something is invalid, return None."""

            log_page_number, offset_in_page = self.lsn_to_offset(current_lsn)
            if log_page_number == 0:
                return

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                return

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset_in_page, self.record_header_length)
            except LogFileException:
                return

            client_previous_lsn = log_record_header.get_client_previous_lsn()
            candidate_log_page_number, candidate_offset_in_page = self.lsn_to_offset(client_previous_lsn)

            if candidate_log_page_number == 0:
                return

            candidate_log_record_page = self.get_log_record_page_by_number(candidate_log_page_number)
            if candidate_log_record_page is not None:
                try:
                    candidate_log_record_header = candidate_log_record_page.get_log_record_at_offset(
                        candidate_offset_in_page, self.record_header_length)
                except LogFileException:
                    pass
                else:
                    if candidate_log_record_header.get_this_lsn() == client_previous_lsn:
                        return client_previous_lsn

        def find_undo_next_lsn(current_lsn):
            """Give a current LSN, find, validate and return the undo next LSN for the same client. If there is none or something is invalid, return None."""

            log_page_number, offset_in_page = self.lsn_to_offset(current_lsn)
            if log_page_number == 0:
                return

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                return

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset_in_page, self.record_header_length)
            except LogFileException:
                return

            client_undo_next_lsn = log_record_header.get_client_undo_next_lsn()
            candidate_log_page_number, candidate_offset_in_page = self.lsn_to_offset(client_undo_next_lsn)

            if candidate_log_page_number == 0:
                return

            candidate_log_record_page = self.get_log_record_page_by_number(candidate_log_page_number)
            if candidate_log_record_page is not None:
                try:
                    candidate_log_record_header = candidate_log_record_page.get_log_record_at_offset(
                        candidate_offset_in_page, self.record_header_length)
                except LogFileException:
                    pass
                else:
                    if candidate_log_record_header.get_this_lsn() == client_undo_next_lsn:
                        return client_undo_next_lsn

        # Find new LSNs by walking over client data.
        for lsn in self.lsns.copy():
            current_lsn = lsn

            while True:
                current_lsn = find_next_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

        # Find new LSNs by looking for previous LSNs recorded for the same client.
        for lsn in self.lsns.copy():
            current_lsn = lsn

            while True:
                current_lsn = find_previous_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

        # Find new LSNs by looking for undo next LSNs for the same client.
        for lsn in self.lsns.copy():
            current_lsn = lsn

            while True:
                current_lsn = find_undo_next_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

        # Find new LSNs by looking in NTFS restart areas.
        new_lsns = set()

        for lsn in self.lsns.copy():
            client_data = self.get_client_data(lsn)
            if client_data.record_type == LfsClientRestart:
                try:
                    ntfs_restart_area = client_data.data_decoded()
                except ClientException:
                    continue

                # Adjust the cluster size.
                cluster_size = ntfs_restart_area.get_bytes_per_cluster()
                if cluster_size is not None and cluster_size > 0 and cluster_size % 512 == 0:
                    self.cluster_size = cluster_size

                # Adjust the version-specific configuration.
                self.use_dumps = True
                major_version = ntfs_restart_area.get_major_version()
                if major_version >= 1:
                    if major_version >= 2:
                        self.use_dumps = False

                    self.oat_version = 1
                else:
                    self.oat_version = 0

                new_lsns.add(ntfs_restart_area.get_start_of_checkpoint_lsn())
                new_lsns.add(ntfs_restart_area.get_open_attribute_table_lsn())
                new_lsns.add(ntfs_restart_area.get_attribute_names_lsn())
                new_lsns.add(ntfs_restart_area.get_dirty_page_table_lsn())
                new_lsns.add(ntfs_restart_area.get_transaction_table_lsn())

        for lsn in new_lsns:
            # Validate a new LSN.
            log_page_number, offset = self.lsn_to_offset(lsn)
            if log_page_number == 0:
                continue

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                continue

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset, self.record_header_length)
            except LogFileException:
                continue

            if lsn == log_record_header.get_this_lsn():
                self.lsns.add(lsn)
            else:
                continue

            # Search for new LSNs.
            current_lsn = lsn
            while True:
                current_lsn = find_next_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

            current_lsn = lsn
            while True:
                current_lsn = find_previous_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

            current_lsn = lsn
            while True:
                current_lsn = find_undo_next_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

        self.lsns = sorted(self.lsns)  # Finally, store a sorted list of LSNs.

    def sort_lsns(self):
        """Group log sequence numbers (LSNs) by their client IDs."""

        self.lsns_sorted = dict()

        for current_lsn in self.lsns:
            log_page_number, offset_in_page = self.lsn_to_offset(current_lsn)
            log_record_page = self.get_log_record_page_by_number(log_page_number)
            log_record_header = log_record_page.get_log_record_at_offset(offset_in_page, self.record_header_length)

            client_id = log_record_header.get_client_id()
            if client_id not in self.lsns_sorted.keys():
                self.lsns_sorted[client_id] = [current_lsn]
            else:
                self.lsns_sorted[client_id].append(current_lsn)

        for client_id in self.lsns_sorted.keys():
            self.lsns_sorted[client_id] = sorted(self.lsns_sorted[client_id])

    def prepare_for_writes(self):
        """Make a writable file object in memory (which is a copy of the original one) and use it for further operations."""

        if self.writable:
            return

        self.file_object.seek(0)

        new_file_object = io.BytesIO()
        shutil.copyfileobj(self.file_object, new_file_object)

        self.old_file_object = self.file_object
        self.file_object = new_file_object
        self.writable = True

    def discard_changes(self):
        """Discard all changes made to a writable file object (in memory) and use the original file object."""

        if not self.writable:
            return

        self.file_object.close()
        self.file_object = self.old_file_object
        self.writable = False

    def apply_tail_page(self):
        """Apply a tail log record page to the log file. The log file version should be 1.1. The file object should be writable."""

        if self.log_version != (1, 1):
            raise RuntimeError('The log file version is not 1.1')

        tail_page_1 = self.get_log_record_page_by_number(2)
        tail_page_2 = self.get_log_record_page_by_number(3)

        if tail_page_1 is None and tail_page_2 is None:  # Nothing to do, both tail log record pages are invalid.
            return

        single_tail_page = None

        if tail_page_1 is None and tail_page_2 is not None:  # Only the second tail log record page is present.
            single_tail_page = tail_page_2
        elif tail_page_2 is None and tail_page_1 is not None:  # Only the first tail log record page is present.
            single_tail_page = tail_page_1
        elif tail_page_1 is not None and tail_page_2 is not None:  # Both tail log record pages are valid, choose the latest one.
            if tail_page_1.get_last_end_lsn() > tail_page_2.get_last_end_lsn():
                single_tail_page = tail_page_1
            else:
                single_tail_page = tail_page_2

        if single_tail_page is not None:  # A proper tail log record page was found.
            file_offset = single_tail_page.get_file_offset_11()

            self.file_object.seek(file_offset)
            self.file_object.write(single_tail_page.buf_protected)

    def apply_fast_pages(self):
        """Apply fast log record pages to the log file. The log file version should be 2.0 or higher. The file object should be writable."""

        if self.log_version[0] < 2:
            raise RuntimeError('The log file version is not 2.x (or higher)')

        # Locate the highest LSN present in the "infinite" area.
        highest_lsn = 0
        for log_record_page in self.log_record_pages(0):
            current_lsn = log_record_page.get_last_lsn()
            if current_lsn > highest_lsn:
                highest_lsn = current_lsn

        fast_pages = []

        i = 2
        while i < 34:
            fast_page = self.get_log_record_page_by_number(i)
            if fast_page is not None and fast_page.get_last_lsn() > highest_lsn:
                fast_pages.append(fast_page)

            i += 1

        fast_pages.sort(key=lambda
            x: x.get_last_lsn())  # Sort the fast pages according to the log sequence numbers they have (fast pages with more recent log data should be written at the end).

        for fast_page in fast_pages:
            file_offset = fast_page.get_file_offset_2x()

            self.file_object.seek(file_offset)
            self.file_object.write(fast_page.buf_protected)

    def log_record_pages(self, mode):
        """This method yields each log record page (LogRecordPage).
        When 'mode' is 0, yield log record pages from the "infinite" (circular) area only.
        When 'mode' is 1, yield each log record page from the whole file (not just from the "infinite" area).
        """

        if mode not in [0, 1]:
            raise ValueError('Invalid mode of operation: {}'.format(mode))

        if mode == 0:
            start_page = self.get_wrapped_log_record_page_number()
        elif mode == 1:
            start_page = 2  # 2 restart pages.

        i = start_page
        while i < self.file_size // self.log_page_size:
            log_page = self.get_log_record_page_by_number(i)

            if log_page is not None:
                yield log_page

            i += 1

    def get_client_data(self, lsn):
        """Get, parse and return client data (ClientData) for a given log sequence number (LSN)."""

        log_page_number, offset_in_page = self.lsn_to_offset(lsn)

        log_record_page = self.get_log_record_page_by_number(log_page_number)
        log_record_header = log_record_page.get_log_record_at_offset(offset_in_page, self.record_header_length)

        client_data_length = log_record_header.get_client_data_length()

        if client_data_length <= self.log_page_size - offset_in_page - self.record_header_length:  # Client data is in the current log record page only.
            client_buf = log_record_header.buf[
                         self.record_header_length: self.record_header_length + client_data_length]
            client_data = ClientData(client_buf, log_record_header.get_record_type(), lsn,
                                     log_record_header.get_transaction_id(), self.cluster_size)

            return client_data

        else:  # Client data is in two or more log record pages.
            client_buf = log_record_header.buf[self.record_header_length:]
            client_data_length_left = client_data_length - len(client_buf)

            while client_data_length_left > 0:
                log_page_number += 1

                # Handle the wrap.
                if log_page_number * self.log_page_size >= self.file_size:
                    log_page_number = self.get_wrapped_log_record_page_number()

                log_record_page = self.get_log_record_page_by_number(log_page_number)
                if log_record_page is None:  # This page is missing in the current state of the log file.
                    break

                if client_data_length_left >= self.log_page_size - self.log_page_data_offset:
                    client_buf_more_data = log_record_page.buf[self.log_page_data_offset:]
                else:
                    client_buf_more_data = log_record_page.buf[
                                           self.log_page_data_offset: self.log_page_data_offset + client_data_length_left]

                client_buf += client_buf_more_data

                client_data_length_left -= len(client_buf_more_data)

            client_data = ClientData(client_buf, log_record_header.get_record_type(), lsn,
                                     log_record_header.get_transaction_id(), self.cluster_size)

            return client_data

    def parse_ntfs_records(self, recover_log_data=True):
        """This method yields NTFS client data (NTFSRestartArea and NTFSLogRecord) from each log record found in the log file.
        If 'recover_log_data' is True, apply a tail log record page or fast log record pages before parsing the log data.
        """

        if recover_log_data:
            self.prepare_for_writes()

            if self.log_version == (1, 1):
                self.apply_tail_page()
            elif self.log_version[0] >= 2:
                self.apply_fast_pages()

        self.collect_lsns()
        self.sort_lsns()

        for client_id in self.lsns_sorted.keys():
            latest_oat_dump = None
            old_oat_dump = None
            latest_an_dump = None
            old_an_dump = None

            oat_dump_count = 0
            an_dump_count = 0

            oat = dict()

            for lsn in self.lsns_sorted[client_id]:
                client_data = self.get_client_data(lsn)
                try:
                    client_data_decoded = client_data.data_decoded()
                except NotImplementedError:
                    pass
                else:
                    if type(client_data_decoded) is NTFSLogRecord:
                        redo_op = client_data_decoded.get_redo_operation()
                        if redo_op == OpenNonresidentAttribute:
                            attribute_name = client_data_decoded.get_undo_data().decode('utf-16le', errors='replace')

                            oat_index = client_data_decoded.get_target_attribute()
                            oat_entry_data = client_data_decoded.get_redo_data()

                            try:
                                if self.oat_version >= 1:
                                    oat_entry = OpenAttributeEntry1(oat_entry_data)
                                else:
                                    oat_entry = OpenAttributeEntry0(oat_entry_data)
                            except ClientException:
                                # Something is wrong, clear the open attribute table.
                                oat = dict()
                            else:
                                oat[oat_index] = (oat_entry.get_file_reference(), attribute_name)
                        elif self.use_dumps and redo_op == OpenAttributeTableDump:
                            old_oat_dump = latest_oat_dump
                            latest_oat_dump = OpenAttributeTableDumpParser(client_data_decoded.get_redo_data(),
                                                                           self.oat_version)
                            oat_dump_count += 1
                        elif self.use_dumps and redo_op == AttributeNamesDump:
                            old_an_dump = latest_an_dump
                            latest_an_dump = AttributeNamesDumpParser(client_data_decoded.get_redo_data())
                            an_dump_count += 1

                        client_data_decoded.oat = oat.copy()

                        if self.use_dumps and oat_dump_count > 0 and oat_dump_count == an_dump_count:
                            client_data_decoded.oat_dump = latest_oat_dump
                            client_data_decoded.an_dump = latest_an_dump
                        elif self.use_dumps and oat_dump_count > 0 and oat_dump_count != an_dump_count:
                            client_data_decoded.oat_dump = old_oat_dump
                            client_data_decoded.an_dump = old_an_dump
                        else:
                            client_data_decoded.oat_dump = None
                            client_data_decoded.an_dump = None

                    yield client_data_decoded

        if recover_log_data:
            self.discard_changes()

    def __str__(self):
        return 'LogFileParser, file size: {}'.format(self.file_size)


class ClientData(object):
    """This class is used to work with client data."""

    buf = None
    """Client data."""

    record_type = None
    """A record type of this client data."""

    lsn = None
    """A log sequence number (LSN) for this client data."""

    cluster_size = None
    """A number of bytes per cluster."""

    transaction_id = None
    """A transaction ID."""

    def __init__(self, client_data_raw, record_type, log_sequence_number, transaction_id, cluster_size):
        self.buf = client_data_raw
        self.record_type = record_type
        self.lsn = log_sequence_number
        self.transaction_id = transaction_id
        self.cluster_size = cluster_size

    def data_decoded(self):
        """Parse and return decoded data (NTFSRestartArea or NTFSLogRecord). Only the NTFS client is supported."""

        if self.record_type == LfsClientRestart:
            return NTFSRestartArea(self.buf, self.lsn)
        elif self.record_type == LfsClientRecord:
            return NTFSLogRecord(self.buf, self.lsn, self.transaction_id, self.cluster_size)
        else:
            raise NotImplementedError('The following record type is not supported: {}'.format(self.record_type))

    def __str__(self):
        return 'ClientData, length: {}, record type: {}, LSN: {}'.format(len(self.buf), self.record_type, self.lsn)


class NTFSRestartArea(object):
    """This class is used work with an NTFS restart area."""

    buf = None
    """Data of this NTFS restart area."""

    lsn = None
    """A log sequence number (LSN) of an NTFS restart area."""

    def __init__(self, restart_area_raw, lsn):
        self.buf = restart_area_raw
        self.lsn = lsn

        if len(self.buf) < 64:
            raise ClientException('Invalid restart area length: {}'.format(len(self.buf)))

    def get_major_version(self):
        """Get and return the major version of a client."""

        return struct.unpack('<L', self.buf[: 4])[0]

    def get_minor_version(self):
        """Get and return the minor version of a client."""

        return struct.unpack('<L', self.buf[4: 8])[0]

    def get_start_of_checkpoint_lsn(self):
        """Get and return the log sequence number (LSN) of a checkpoint record."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_open_attribute_table_lsn(self):
        """Get and return the log sequence number (LSN) of an open attribute table."""

        return struct.unpack('<Q', self.buf[16: 24])[0]

    def get_attribute_names_lsn(self):
        """Get and return the log sequence number (LSN) of attribute names."""

        return struct.unpack('<Q', self.buf[24: 32])[0]

    def get_dirty_page_table_lsn(self):
        """Get and return the log sequence number (LSN) of a dirty page table."""

        return struct.unpack('<Q', self.buf[32: 40])[0]

    def get_transaction_table_lsn(self):
        """Get and return the log sequence number (LSN) of a transaction table."""

        return struct.unpack('<Q', self.buf[40: 48])[0]

    def get_open_attribute_table_length(self):
        """Get and return the length of an open attribute table."""

        return struct.unpack('<L', self.buf[48: 52])[0]

    def get_attribute_names_length(self):
        """Get and return the length of attribute names."""

        return struct.unpack('<L', self.buf[52: 56])[0]

    def get_dirty_page_table_length(self):
        """Get and return the length of a dirty page table."""

        return struct.unpack('<L', self.buf[56: 60])[0]

    def get_transaction_table_length(self):
        """Get and return the length of a transaction table."""

        return struct.unpack('<L', self.buf[60: 64])[0]

    def get_usn_journal_restart_offset(self):
        """Get and return the absolute offset used as a starting point for checking update sequence number (USN) change journal records."""

        if len(self.buf) >= 72:
            return struct.unpack('<q', self.buf[64: 72])[0]

    def get_last_lsn(self):
        """Get and return the last value (as of the previous volume restart) of the "current" log sequence number (LSN)."""

        if len(self.buf) >= 80:
            return struct.unpack('<Q', self.buf[72: 80])[0]

    def get_bytes_per_cluster(self):
        """Get and return the number of bytes per cluster."""

        if len(self.buf) >= 84:
            return struct.unpack('<L', self.buf[80: 84])[0]

    def get_usn_journal_reference(self):
        """Get and return the file record segment (FRS) reference for the update sequence number (USN) change journal."""

        if len(self.buf) >= 96:
            return struct.unpack('<Q', self.buf[88: 96])[0]

    def get_usn_base(self):
        """Get and return the base offset for the update sequence number (USN) change journal.
        Since the $J data stream of this journal can be very large, this value is used to rebase offsets for high-level API functions (making them relative from the base offset).
        """

        if len(self.buf) >= 104:
            return struct.unpack('<q', self.buf[96: 104])[0]

    def get_oldest_lsn(self):
        """Get and return the oldest log sequence number (LSN) required by this client (as of the checkpoint)."""

        if len(self.buf) >= 112:
            return struct.unpack('<Q', self.buf[104: 112])[0]

    def __str__(self):
        return 'NTFSRestartArea (LSN: {}), version: {}.{}, start of checkpoint LSN: {}'.format(self.lsn,
                                                                                               self.get_major_version(),
                                                                                               self.get_minor_version(),
                                                                                               self.get_start_of_checkpoint_lsn())


class NTFSLogRecord(object):
    """This class is used work with an NTFS log record."""

    buf = None
    """Data of this NTFS log record."""

    lsn = None
    """A log sequence number (LSN) of this NTFS log record."""

    oat = None
    """An open attribute table (it is set externally in the 'parse_ntfs_records' method of the LogFileParser class)."""

    oat_dump = None
    """An open attribute table dump (as raw bytes; it is set externally in the 'parse_ntfs_records' method of the LogFileParser class)."""

    an_dump = None
    """An attribute names dump (as raw bytes; it is set externally in the 'parse_ntfs_records' method of the LogFileParser class)."""

    transaction_id = None
    """A transaction ID."""

    def __init__(self, log_record_raw, lsn, transaction_id, cluster_size):
        self.buf = log_record_raw
        self.lsn = lsn
        self.transaction_id = transaction_id

        self.cluster_size = cluster_size

        if len(self.buf) < 32:
            raise ClientException('Invalid log record length: {}'.format(len(self.buf)))

    def get_redo_operation(self):
        """Get and return the redo operation code (as an integer)."""

        return struct.unpack('<H', self.buf[0: 2])[0]

    def get_undo_operation(self):
        """Get and return the undo operation code (as an integer)."""

        return struct.unpack('<H', self.buf[2: 4])[0]

    def get_redo_offset(self):
        """Get and return the offset of an undo record."""

        offset = struct.unpack('<H', self.buf[4: 6])[0]
        if 0 < offset < 32:
            raise ClientException('Invalid redo offset: {}'.format(offset))

        return offset

    def get_redo_length(self):
        """Get and return the length of an undo record."""

        return struct.unpack('<H', self.buf[6: 8])[0]

    def get_undo_offset(self):
        """Get and return the offset of an undo record."""

        offset = struct.unpack('<H', self.buf[8: 10])[0]
        if 0 < offset < 32:
            raise ClientException('Invalid undo offset: {}'.format(offset))

        return offset

    def get_undo_length(self):
        """Get and return the length of an undo record."""

        return struct.unpack('<H', self.buf[10: 12])[0]

    def get_target_attribute(self):
        """Get and return the open attribute table index for this update."""

        return struct.unpack('<H', self.buf[12: 14])[0]

    def get_lcns_to_follow(self):
        """Get and return the number of logical cluster numbers (LCNs) in use at the end of the header."""

        return struct.unpack('<H', self.buf[14: 16])[0]

    def get_record_offset(self):
        """Get and return the record offset."""

        return struct.unpack('<H', self.buf[16: 18])[0]

    def get_attribute_offset(self):
        """Get and return the attribute offset."""

        return struct.unpack('<H', self.buf[18: 20])[0]

    def get_cluster_block_offset(self):
        """Get and return the cluster block offset (this is a number of 512-byte blocks)."""

        return struct.unpack('<H', self.buf[20: 22])[0]

    def get_target_block_size(self):
        """For operations on the $MFT file, this field is set to the size (in bytes) of the file record segment (FRS) divided by 512.
        If the attribute is $INDEX_ALLOCATION, this field is set to the size (in bytes) of the index record divided by 512.
        In other cases, this field is set to 0.
        """

        return struct.unpack('<H', self.buf[22: 24])[0]

    def get_target_vcn(self):
        """Get and return the target virtual cluster number (VCN)."""

        return struct.unpack('<q', self.buf[24: 32])[0]

    def get_lcns_for_page(self):
        """Get and return the array of logical cluster numbers (LCNs) as a list of integers."""

        lcns = []

        i = 0
        while i < self.get_lcns_to_follow():
            lcn_buf = self.buf[32 + i * 8: 40 + i * 8]
            if len(lcn_buf) != 8:
                raise ClientException('Invalid (truncated) log record length: {}'.format(len(self.buf)))

            lcn = struct.unpack('<q', lcn_buf)[0]

            lcns.append(lcn)

            i += 1

        return lcns

    def get_redo_data(self):
        """Get and return redo data."""

        redo_length = self.get_redo_length()
        if redo_length == 0:
            return b''

        redo_offset = self.get_redo_offset()
        return self.buf[redo_offset: redo_offset + redo_length]

    def get_undo_data(self):
        """Get and return undo data."""

        undo_length = self.get_undo_length()
        if undo_length == 0:
            return b''

        undo_offset = self.get_undo_offset()
        return self.buf[undo_offset: undo_offset + undo_length]

    def calculate_mft_target_number(self):
        """Calculate and return the number of a target file record segment (or None, if not applicable).
        Note: this is for resident updates only.
        """

        if self.get_redo_operation() in LOGGED_RESIDENT_UPDATES or self.get_undo_operation() in LOGGED_RESIDENT_UPDATES:
            target_vcn = self.get_target_vcn()
            cluster_block_offset = self.get_cluster_block_offset()

            target_block_size = self.get_target_block_size()
            if target_block_size > 0:
                frs_size = target_block_size * 512
            else:
                frs_size = FILE_RECORD_SEGMENT_SIZE_DEFAULT

            return (target_vcn * self.cluster_size + cluster_block_offset * 512) // frs_size

    def calculate_offset_in_target(self):
        """Calculate and return the offset within target data (or None, if not applicable)."""

        if self.get_redo_operation() in LOGGED_RESIDENT_UPDATES or self.get_undo_operation() in LOGGED_RESIDENT_UPDATES:
            return self.get_attribute_offset() + self.get_record_offset()

        elif self.get_redo_operation() in LOGGED_NONRESIDENT_UPDATES or self.get_undo_operation() in LOGGED_NONRESIDENT_UPDATES:
            return self.get_target_vcn() * self.cluster_size + self.get_cluster_block_offset() * 512 + self.get_attribute_offset() + self.get_record_offset()

    def calculate_mft_target_reference_and_name(self):
        """Calculate and return the reference to a target file record segment (FRS) and a target attribute name as a tuple (or None, if not applicable or not found).
        Note: this is for nonresident updates present in an open attribute table.
        """

        if self.oat is None:
            return

        if self.get_redo_operation() in LOGGED_NONRESIDENT_UPDATES or self.get_undo_operation() in LOGGED_NONRESIDENT_UPDATES:
            target_attribute = self.get_target_attribute()
            if target_attribute in self.oat.keys():
                return self.oat[target_attribute]

            # If not found, try the table dumps.
            if self.oat_dump is not None and self.an_dump is not None:
                try:
                    target_file_reference = self.oat_dump.find_file_reference_by_index(target_attribute)
                except ClientException:
                    pass
                else:
                    target_attribute_name = self.an_dump.find_name_by_index(target_attribute)
                    return (target_file_reference, target_attribute_name)

    def __str__(self):
        if self.get_redo_offset() > 0:
            redo_length = self.get_redo_length()
        else:
            redo_length = 0

        if self.get_undo_offset() > 0:
            undo_length = self.get_undo_length()
        else:
            undo_length = 0

        if redo_length > 0 and redo_length == undo_length and self.get_redo_offset() == self.get_undo_offset():
            redo_undo_status = '=='
        else:
            redo_undo_status = '!='

        frs_number = self.calculate_mft_target_number()
        if frs_number is not None:
            target_spec = '{} (FRS number)'.format(frs_number)
        else:
            frs_reference_and_attribute_name = self.calculate_mft_target_reference_and_name()
            if frs_reference_and_attribute_name is not None:
                frs_reference, attribute_name = frs_reference_and_attribute_name
                target_spec = '{}/{} (FRS reference / attribute name)'.format(frs_reference, attribute_name)
            else:
                target_spec = 'unknown'

        target_offset = self.calculate_offset_in_target()
        if target_offset is None:
            target_offset = 'unknown'

        return 'NTFSLogRecord (LSN: {}), redo operation: {} (length: {}), undo operation: {} (length: {}), redo {} undo, target: {}, offset in target: {}'.format(
            self.lsn, ResolveNTFSOperation(self.get_redo_operation()), redo_length,
            ResolveNTFSOperation(self.get_undo_operation()), undo_length, redo_undo_status, target_spec, target_offset)


class OpenAttributeEntry1(object):
    """This class is used to work with the OPEN_ATTRIBUTE_ENTRY structure (version: 1.0+)."""

    buf = None
    """Data of this OPEN_ATTRIBUTE_ENTRY structure."""

    def __init__(self, open_attribute_entry_raw):
        self.buf = open_attribute_entry_raw

        if len(self.buf) < 40:
            raise ClientException('Invalid open attribute entry length: {}'.format(len(self.buf)))

    def get_allocated_or_next_free(self):
        """Get the free link (this value is 0xFFFFFFFF if this entry is allocated)."""

        return struct.unpack('<L', self.buf[0: 4])[0]

    def get_bytes_per_index_buffer(self):
        """Get and return the number of bytes per index buffer."""

        return struct.unpack('<L', self.buf[4: 8])[0]

    def get_attribute_type_code(self):
        """Get and return the attribute type code."""

        return struct.unpack('<L', self.buf[8: 12])[0]

    # At least one source states that there is a field between these two fields, but there is none.

    def get_file_reference(self):
        """Get and return the file reference."""

        return struct.unpack('<Q', self.buf[16: 24])[0]

    def get_lsn_of_open_record(self):
        """Get and return the log sequence number (LSN) of an open log record (OpenNonresidentAttribute)."""

        return struct.unpack('<Q', self.buf[24: 32])[0]


# No field for the pointer is supported (because it is valid for a running machine only).


class OpenAttributeEntry0(object):
    """This class is used to work with the OPEN_ATTRIBUTE_ENTRY structure (version: 0.0)."""

    buf = None
    """Data of this OPEN_ATTRIBUTE_ENTRY structure."""

    def __init__(self, open_attribute_entry_raw):
        self.buf = open_attribute_entry_raw

        if len(self.buf) < 44:
            raise ClientException('Invalid open attribute entry length: {}'.format(len(self.buf)))

    def get_allocated_or_next_free(self):
        """Get the free link (this value is 0xFFFFFFFF if this entry is allocated)."""

        return struct.unpack('<L', self.buf[0: 4])[0]

    def get_table_index(self):
        """Get and return the table index (offset) of an original (version: 1.0+) entry (in a table in memory)."""

        return struct.unpack('<L', self.buf[4: 8])[0]

    def get_file_reference(self):
        """Get and return the file reference."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_lsn_of_open_record(self):
        """Get and return the log sequence number (LSN) of an open log record (OpenNonresidentAttribute)."""

        return struct.unpack('<Q', self.buf[16: 24])[0]

    def get_attribute_type_code(self):
        """Get and return the attribute type code."""

        return struct.unpack('<L', self.buf[28: 32])[0]

    def get_bytes_per_index_buffer(self):
        """Get and return the number of bytes per index buffer."""

        return struct.unpack('<L', self.buf[40: 44])[0]


class AttributeNameEntry(object):
    """This class is used to work with the ATTRIBUTE_NAME_ENTRY structure."""

    buf = None
    """Data of this ATTRIBUTE_NAME_ENTRY structure."""

    def __init__(self, attribute_name_entry_raw):
        self.buf = attribute_name_entry_raw

        if len(self.buf) < 8:
            raise ClientException('Invalid attribute name entry length: {}'.format(len(self.buf)))

    def get_index(self):
        """Get and return the table index (offset) of a corresponding entry in the open attribute table."""

        return struct.unpack('<H', self.buf[0: 2])[0]

    def get_name_length(self):
        """Get and return the name length."""

        return struct.unpack('<H', self.buf[2: 4])[0]

    def get_name(self):
        """Get and return the name."""

        name_buf = self.buf[4: 4 + self.get_name_length()]
        name = name_buf.decode('utf-16le', errors='replace')

        return name

    def calculate_sizeof(self):
        """Calculate and return the size of this entry."""

        return 4 + self.get_name_length() + 2  # Include the terminating null character.


class AttributeNamesDumpParser(object):
    """This class is used to work with the array of ATTRIBUTE_NAME_ENTRY structures."""

    buf = None
    """Data of this array."""

    def __init__(self, attribute_names_dump_raw):
        self.buf = attribute_names_dump_raw

    def find_name_by_index(self, index):
        """Find and return the name by its index (or None, if not found)."""

        pos = 0
        while pos < len(self.buf):
            attribute_name_entry_buf = self.buf[pos:]

            try:
                attribute_name_entry = AttributeNameEntry(attribute_name_entry_buf)
            except ClientException:
                break

            attribute_name_entry_index = attribute_name_entry.get_index()
            if attribute_name_entry_index == index:
                return attribute_name_entry.get_name()

            pos += attribute_name_entry.calculate_sizeof()


class OpenAttributeTableDumpParser(object):
    """This class is used to work with the array of OPEN_ATTRIBUTE_ENTRY structures."""

    buf = None
    """Data of this array."""

    oat_class = None
    """Either OpenAttributeEntry0 or OpenAttributeEntry1."""

    def __init__(self, attribute_names_dump_raw, version):
        self.buf = attribute_names_dump_raw

        if version >= 1:
            self.oat_class = OpenAttributeEntry1
        else:
            self.oat_class = OpenAttributeEntry0

    def find_file_reference_by_index(self, index):
        """Find and return the file reference using the entry index."""

        if index >= len(self.buf):
            raise ClientException('Invalid index: {}'.format(index))

        open_attribute_entry_buf = self.buf[index:]
        open_attribute_entry = self.oat_class(open_attribute_entry_buf)

        return open_attribute_entry.get_file_reference()

# dfir_ntfs: an NTFS parser for digital forensics & incident response
# (c) <NAME>
#
# This module implements an interface to work with the $LogFile journal.

import struct
import io
import shutil
import binascii

UPDATE_SEQUENCE_STRIDE = 512
PAGE_SIZE = 4096  # This is the size of a single memory page.

CLUSTER_SIZE_DEFAULT = 4096
FILE_RECORD_SEGMENT_SIZE_DEFAULT = 1024

# Values for the restart area:
LFS_NO_CLIENT = 0xFFFF  # No index in the array of client records.

# Flags for the restart area:
RESTART_SINGLE_PAGE_IO = 0x1  # Log entries are written one by one.
RESTART_UNKNOWN_NAME_2 = 0x2  # The new restart area has been written to the log file.

# Flags for the log record page:
LOG_PAGE_LOG_RECORD_END = 0x1  # This page contains the end of a log record.
LOG_PAGE_UNKNOWN_2 = 0x2  # This page contains a client restart area (set when the log file version is 2.0 or higher).

# Flags for the log record header:
LOG_RECORD_MULTI_PAGE = 0x1  # This record was written to multiple pages.
LOG_RECORD_UNKNOWN_NAME_2 = 0x2  # This record contains no redo buffer.
LOG_RECORD_UNKNOWN_NAME_4 = 0x4  # This record contains no undo buffer.

# Values for the log record types:
LfsClientRecord = 0x1
LfsClientRestart = 0x2

# Operations for the NTFS client:
Noop = 0x00
CompensationLogRecord = 0x01
InitializeFileRecordSegment = 0x02  # [1]
DeallocateFileRecordSegment = 0x03  # [1]
WriteEndOfFileRecordSegment = 0x04  # [1]
CreateAttribute = 0x05  # [1]
DeleteAttribute = 0x06  # [1]
UpdateResidentValue = 0x07  # [1]
UpdateNonresidentValue = 0x08  # [2]
UpdateMappingPairs = 0x09  # [1]
DeleteDirtyClusters = 0x0A
SetNewAttributeSizes = 0x0B  # [1]
AddIndexEntryRoot = 0x0C  # [1]
DeleteIndexEntryRoot = 0x0D  # [1]
AddIndexEntryAllocation = 0x0E  # [2]
DeleteIndexEntryAllocation = 0x0F  # [2]
WriteEndOfIndexBuffer = 0x10  # [2]
SetIndexEntryVcnRoot = 0x11  # [1]
SetIndexEntryVcnAllocation = 0x12  # [2]
UpdateFileNameRoot = 0x13  # [1]
UpdateFileNameAllocation = 0x14  # [2]
SetBitsInNonresidentBitMap = 0x15  # [2]
ClearBitsInNonresidentBitMap = 0x16  # [2]
HotFix = 0x17
EndTopLevelAction = 0x18
PrepareTransaction = 0x19
CommitTransaction = 0x1A
ForgetTransaction = 0x1B
OpenNonresidentAttribute = 0x1C
OpenAttributeTableDump = 0x1D
AttributeNamesDump = 0x1E
DirtyPageTableDump = 0x1F
TransactionTableDump = 0x20
UpdateRecordDataRoot = 0x21  # [1]
UpdateRecordDataAllocation = 0x22  # [2]
UpdateRelativeDataIndex = 0x23  # [1]
UpdateRelativeDataAllocation = 0x24  # [2]
ZeroEndOfFileRecord = 0x25  # [1]

# [1]: this is an update to the resident ($MFT) data.

LOGGED_RESIDENT_UPDATES = [InitializeFileRecordSegment, DeallocateFileRecordSegment, WriteEndOfFileRecordSegment,
                           CreateAttribute, DeleteAttribute, UpdateResidentValue, UpdateMappingPairs,
                           SetNewAttributeSizes, AddIndexEntryRoot, DeleteIndexEntryRoot, SetIndexEntryVcnRoot,
                           UpdateFileNameRoot, UpdateRecordDataRoot, UpdateRelativeDataIndex, ZeroEndOfFileRecord]

# [2]: this is an update to the nonresident data (which is referenced by an entry in the open attribute table).

LOGGED_NONRESIDENT_UPDATES = [UpdateNonresidentValue, AddIndexEntryAllocation, DeleteIndexEntryAllocation,
                              WriteEndOfIndexBuffer, SetIndexEntryVcnAllocation, UpdateFileNameAllocation,
                              SetBitsInNonresidentBitMap, ClearBitsInNonresidentBitMap, UpdateRecordDataAllocation,
                              UpdateRelativeDataAllocation]

NTFSOperations = {
    Noop: 'Noop',
    CompensationLogRecord: 'CompensationLogRecord',
    InitializeFileRecordSegment: 'InitializeFileRecordSegment',
    DeallocateFileRecordSegment: 'DeallocateFileRecordSegment',
    WriteEndOfFileRecordSegment: 'WriteEndOfFileRecordSegment',
    CreateAttribute: 'CreateAttribute',
    DeleteAttribute: 'DeleteAttribute',
    UpdateResidentValue: 'UpdateResidentValue',
    UpdateNonresidentValue: 'UpdateNonresidentValue',
    UpdateMappingPairs: 'UpdateMappingPairs',
    DeleteDirtyClusters: 'DeleteDirtyClusters',
    SetNewAttributeSizes: 'SetNewAttributeSizes',
    AddIndexEntryRoot: 'AddIndexEntryRoot',
    DeleteIndexEntryRoot: 'DeleteIndexEntryRoot',
    AddIndexEntryAllocation: 'AddIndexEntryAllocation',
    DeleteIndexEntryAllocation: 'DeleteIndexEntryAllocation',
    WriteEndOfIndexBuffer: 'WriteEndOfIndexBuffer',
    SetIndexEntryVcnRoot: 'SetIndexEntryVcnRoot',
    SetIndexEntryVcnAllocation: 'SetIndexEntryVcnAllocation',
    UpdateFileNameRoot: 'UpdateFileNameRoot',
    UpdateFileNameAllocation: 'UpdateFileNameAllocation',
    SetBitsInNonresidentBitMap: 'SetBitsInNonresidentBitMap',
    ClearBitsInNonresidentBitMap: 'ClearBitsInNonresidentBitMap',
    HotFix: 'HotFix',
    EndTopLevelAction: 'EndTopLevelAction',
    PrepareTransaction: 'PrepareTransaction',
    CommitTransaction: 'CommitTransaction',
    ForgetTransaction: 'ForgetTransaction',
    OpenNonresidentAttribute: 'OpenNonresidentAttribute',
    OpenAttributeTableDump: 'OpenAttributeTableDump',
    AttributeNamesDump: 'AttributeNamesDump',
    DirtyPageTableDump: 'DirtyPageTableDump',
    TransactionTableDump: 'TransactionTableDump',
    UpdateRecordDataRoot: 'UpdateRecordDataRoot',
    UpdateRecordDataAllocation: 'UpdateRecordDataAllocation',
    UpdateRelativeDataIndex: 'UpdateRelativeDataIndex',
    UpdateRelativeDataAllocation: 'UpdateRelativeDataAllocation',
    ZeroEndOfFileRecord: 'ZeroEndOfFileRecord'
}


def ResolveNTFSOperation(NTFSOperationCode):
    """Convert an NTFS operation code to a string."""

    if NTFSOperationCode in NTFSOperations.keys():
        return NTFSOperations[NTFSOperationCode]

    return hex(NTFSOperationCode)


def UnprotectSectors(Buffer):
    """Apply an update sequence array (USA) to multiple sectors (as a bytearray object) and return the (resulting buffer, end_of_usa_offset) tuple."""

    if len(Buffer) < 2 * UPDATE_SEQUENCE_STRIDE or len(Buffer) % UPDATE_SEQUENCE_STRIDE != 0:
        raise UpdateSequenceArrayException('Invalid input (not two or more sectors)')

    usa_offset, usa_size = struct.unpack('<HH', Buffer[4: 8])

    if usa_offset < 8 or usa_offset > UPDATE_SEQUENCE_STRIDE - 6:
        raise UpdateSequenceArrayException('Invalid update sequence array offset: {}'.format(usa_offset))

    if usa_size < 2 or (usa_size - 1) * UPDATE_SEQUENCE_STRIDE != len(Buffer):
        raise UpdateSequenceArrayException('Invalid update sequence array size: {}'.format(usa_size))

    if usa_offset + usa_size * 2 >= len(Buffer):
        raise UpdateSequenceArrayException(
            'Invalid update sequence array offset and size: {}, {}'.format(usa_offset, usa_size))

    if Buffer[
       usa_offset + 4: usa_offset + 8] == b'CRC4':  # Check if integrity of data in this block is protected using the CRC32 value (for DAX volumes).
        crc32_recorded_raw = Buffer[usa_offset: usa_offset + 4]
        crc32_recorded = struct.unpack('<L', crc32_recorded_raw)[0]

        buffer_modified = Buffer[:]
        buffer_modified[usa_offset: usa_offset + usa_size * 2] = b'\x00' * (usa_size * 2)

        crc32_calculated = binascii.crc32(buffer_modified)

        if crc32_recorded == crc32_calculated:
            end_of_usa_offset = usa_offset + usa_size * 2
            return (Buffer, end_of_usa_offset)

    # If the checksum value is invalid, treat this block as a usual one.

    sequence_number_in_usa_bytes = Buffer[usa_offset: usa_offset + 2]

    i = 1  # Skip the first element (sequence_number_in_usa_bytes).
    while i < usa_size:
        offset_in_usa = i * 2
        update_bytes = Buffer[usa_offset + offset_in_usa: usa_offset + offset_in_usa + 2]

        offset_in_buf = i * UPDATE_SEQUENCE_STRIDE - 2
        sequence_number_in_sector_bytes = Buffer[offset_in_buf: offset_in_buf + 2]

        if sequence_number_in_usa_bytes != sequence_number_in_sector_bytes:
            raise UpdateSequenceArrayException(
                'Invalid sequence number in the buffer, relative offset: {}'.format(offset_in_buf))

        Buffer[offset_in_buf] = update_bytes[0]
        Buffer[offset_in_buf + 1] = update_bytes[1]

        i += 1

    end_of_usa_offset = usa_offset + usa_size * 2
    return (Buffer, end_of_usa_offset)


class LogFileException(Exception):
    """This is a top-level exception for this module."""

    def __init__(self, value):
        self._value = value

    def __str__(self):
        return repr(self._value)


class UpdateSequenceArrayException(LogFileException):
    """This exception is raised when something is wrong with an update sequence array."""

    pass


class RestartPageException(LogFileException):
    """This exception is raised when something is wrong with a restart page."""

    pass


class EmptyLogFileException(RestartPageException):
    """This exception is raised when the log file is empty."""

    pass


class RestartAreaException(LogFileException):
    """This exception is raised when something is wrong with a restart area."""

    pass


class ClientRecordException(LogFileException):
    """This exception is raised when something is wrong with a client record."""

    pass


class LogRecordPageException(LogFileException):
    """This exception is raised when something is wrong with a log record page."""

    pass


class ClientException(LogFileException):
    """This exception is raised when something is wrong with client data."""

    pass


class RestartPage(object):
    """This class is used to work with a restart page."""

    buf = None
    """Data of this restart page with updates from an update sequence array (USA) applied."""

    def __init__(self, restart_page_raw):
        self.buf, __ = UnprotectSectors(bytearray(restart_page_raw))

        signature = self.buf[: 4]
        if signature not in [b'RSTR', b'CHKD', b'BAAD']:
            raise RestartPageException('Invalid signature: {}'.format(signature))

        major_version = self.get_major_version()
        minor_version = self.get_minor_version()

        if (major_version, minor_version) not in [(1, 1), (2, 0), (3, 0)]:
            version_str = '{}.{}'.format(major_version, minor_version)
            raise NotImplementedError('Version not supported: {}'.format(version_str))

        system_page_size = self.get_system_page_size()
        if system_page_size < 1024 or system_page_size % 512 != 0:
            raise RestartPageException('Invalid system page size: {}'.format(system_page_size))

        if system_page_size != PAGE_SIZE:
            raise NotImplementedError('The following system page size is not supported: {}'.format(system_page_size))

        log_page_size = self.get_log_page_size()
        if log_page_size < 1024 or log_page_size % 512 != 0:
            raise RestartPageException('Invalid log page size: {}'.format(log_page_size))

        if log_page_size != PAGE_SIZE:
            raise NotImplementedError('The following log page size is not supported: {}'.format(log_page_size))

        restart_offset = self.get_restart_offset()
        if restart_offset == 0 or restart_offset % 8 != 0 or restart_offset >= len(
                self.buf) or restart_offset >= system_page_size:
            raise RestartPageException('Invalid restart offset: {}'.format(restart_offset))

    def get_chkdsk_lsn(self):
        """Get and return the last log sequence number (LSN) found during the chkdsk scan."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_system_page_size(self):
        """Get and return the system page size. This is the size of a restart page."""

        return struct.unpack('<L', self.buf[16: 20])[0]

    def get_log_page_size(self):
        """Get and return the log page size. This is the size of a log record page."""

        return struct.unpack('<L', self.buf[20: 24])[0]

    def get_restart_offset(self):
        """Get and return the restart area offset (relative from the start of this structure)."""

        return struct.unpack('<H', self.buf[24: 26])[0]

    def get_restart_area(self):
        """Get, parse and return the restart area (as a RestartArea object)."""

        return RestartArea(self.buf[self.get_restart_offset():], self.get_system_page_size())

    def get_major_version(self):
        """Get and return the major version of the log file."""

        return struct.unpack('<H', self.buf[28: 30])[0]

    def get_minor_version(self):
        """Get and return the minor version of the log file."""

        return struct.unpack('<H', self.buf[26: 28])[0]

    def __str__(self):
        return 'RestartPage, version: {}.{}, system page size: {}, log page size: {}'.format(self.get_major_version(),
                                                                                             self.get_minor_version(),
                                                                                             self.get_system_page_size(),
                                                                                             self.get_log_page_size())


class RestartArea(object):
    """This class is used to work with a restart area."""

    buf = None
    """Data of this restart area."""

    def __init__(self, restart_area_raw, system_page_size):
        self.buf = restart_area_raw

        client_array_offset = self.get_client_array_offset()
        if client_array_offset == 0 or client_array_offset % 8 != 0 or client_array_offset >= len(
                self.buf) or client_array_offset >= system_page_size:
            raise RestartAreaException('Invalid client array offset: {}'.format(client_array_offset))

        log_clients = self.get_log_clients()
        client_free_index = self.get_client_free_list()
        client_in_use_index = self.get_client_in_use_list()

        if client_free_index != LFS_NO_CLIENT and client_free_index >= log_clients:
            raise RestartAreaException('Invalid client index (free): {}'.format(client_free_index))

        if client_in_use_index != LFS_NO_CLIENT and client_in_use_index >= log_clients:
            raise RestartAreaException('Invalid client index (in-use): {}'.format(client_in_use_index))

    def get_current_lsn(self):
        """Get and return the current log sequence number (LSN)."""

        return struct.unpack('<Q', self.buf[0: 8])[0]

    def get_log_clients(self):
        """Get and return the number of log clients."""

        return struct.unpack('<H', self.buf[8: 10])[0]

    def get_client_free_list(self):
        """Get and return the index of the free list of client records."""

        return struct.unpack('<H', self.buf[10: 12])[0]

    def get_client_in_use_list(self):
        """Get and return the index of the in-use list of client records."""

        return struct.unpack('<H', self.buf[12: 14])[0]

    def get_flags(self):
        """Get and return the flags (as an integer)."""

        return struct.unpack('<H', self.buf[14: 16])[0]

    def get_sequence_number_bits(self):
        """Get and return the number of bits to use for the sequence number."""

        return struct.unpack('<L', self.buf[16: 20])[0]

    def get_restart_area_length(self):
        """Get and return the size of this restart area."""

        return struct.unpack('<H', self.buf[20: 22])[0]

    def get_client_array_offset(self):
        """Get and return the offset to the array of client records (relative from the start of this structure)."""

        return struct.unpack('<H', self.buf[22: 24])[0]

    def get_file_size(self):
        """Get and return the log file size."""

        return struct.unpack('<q', self.buf[24: 32])[0]

    def get_last_lsn_data_length(self):
        """Get and return the size of data from the last log sequence number (LSN), excluding the header."""

        return struct.unpack('<L', self.buf[32: 36])[0]

    def get_record_header_length(self):
        """Get and return the size of the log page header."""

        return struct.unpack('<H', self.buf[36: 38])[0]

    def get_log_page_data_offset(self):
        """Get and return the log page data offset."""

        return struct.unpack('<H', self.buf[38: 40])[0]

    def get_revision_number(self):
        """Get and return the revision number for this area."""

        return struct.unpack('<L', self.buf[40: 44])[0]

    def clients(self):
        """This method yields client records (ClientRecord objects)."""

        i = 0
        while i < self.get_log_clients():
            curr_pos = i * ClientRecord.sizeof
            buf = self.buf[curr_pos + self.get_client_array_offset():]

            yield ClientRecord(buf)

            i += 1

    def __str__(self):
        return 'RestartArea, clients: {} (free index: {}, in-use index: {}), flags: {}, current LSN: {}, sequence number bits: {}'.format(
            self.get_log_clients(), hex(self.get_client_free_list()), hex(self.get_client_in_use_list()),
            hex(self.get_flags()), self.get_current_lsn(), self.get_sequence_number_bits())


class ClientRecord(object):
    """This class is used to work with a client record."""

    buf = None
    """Data of this client record."""

    sizeof = 160
    """An allocated size of this record."""

    def __init__(self, client_record_raw):
        self.buf = client_record_raw

        client_name_length = self.get_client_name_length()
        if client_name_length > 128:  # Do not cross the 'sizeof' boundary.
            raise ClientRecordException('Invalid name length: {}'.format(client_name_length))

    def get_oldest_lsn(self):
        """Get and return the oldest log sequence number (LSN) required by this client."""

        return struct.unpack('<Q', self.buf[0: 8])[0]

    def get_client_restart_lsn(self):
        """Get and return the log sequence number (LSN) of the latest client restart area."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_prev_client(self):
        """Get and return the index of the previous client record."""

        return struct.unpack('<H', self.buf[16: 18])[0]

    def get_next_client(self):
        """Get and return the index of the next client record."""

        return struct.unpack('<H', self.buf[18: 20])[0]

    def get_sequence_number(self):
        """Get and return the sequence number."""

        return struct.unpack('<H', self.buf[20: 22])[0]

    def get_client_name_length(self):
        """Get and return the name length for this client."""

        return struct.unpack('<L', self.buf[28: 32])[0]

    def get_client_name(self):
        """Get, decode and return the name of this client."""

        name_raw = self.buf[32: 32 + self.get_client_name_length()]
        return name_raw.decode('utf-16le')

    def __str__(self):
        return 'ClientRecord, name: {}, oldest LSN: {}, client restart LSN: {}, sequence number: {}'.format(
            self.get_client_name(), self.get_oldest_lsn(), self.get_client_restart_lsn(), self.get_sequence_number())


class LogRecordPage(object):
    """This class is used to work with a log record page."""

    buf = None
    """Data of this log record page with updates from an update sequence array (USA) applied."""

    buf_protected = None
    """Data of this log record page without updates from an update sequence array (USA) applied."""

    end_of_usa_offset = None
    """Offset to the end of an update sequence array (USA) in the data."""

    data_offset = None
    """Offset to the start of the log data."""

    page_number = None
    """Number of this page in the log file."""

    def __init__(self, log_record_page_raw, log_page_number, data_offset):
        self.buf, self.end_of_usa_offset = UnprotectSectors(bytearray(log_record_page_raw))
        self.buf_protected = log_record_page_raw

        signature = self.buf[: 4]
        if signature not in [b'RCRD', b'CHKD', b'BAAD']:
            raise LogRecordPageException('Invalid signature: {}'.format(signature))

        self.page_number = log_page_number

        if data_offset < 8 or data_offset < self.end_of_usa_offset or data_offset % 8 != 0:
            raise NotImplementedError(
                'The following log data offset is invalid or not supported: {}'.format(data_offset))

        self.data_offset = data_offset

    def get_last_lsn(self):
        """Get and return the highest log sequence number (LSN) in this page."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_file_offset_11(self):
        """Get and return the log file offset for a tail copy. The log file version should be 1.1. This field is not valid for regular log record pages."""

        return struct.unpack('<q', self.buf[8: 16])[0]  # The same offset as above!

    def get_file_offset_2x(self):
        """Get and return the log file offset for a fast copy. The log file version should be 2.0 (or higher). This field is not valid for regular log record pages."""

        if self.end_of_usa_offset <= 60:
            return struct.unpack('<L', self.buf[60: 64])[0]

    def get_flags(self):
        """Get and return the flags (as an integer)."""

        return struct.unpack('<L', self.buf[16: 20])[0]

    def get_page_count(self):
        """Get and return the page count."""

        return struct.unpack('<H', self.buf[20: 22])[0]

    def get_page_position(self):
        """Get and return the page position."""

        return struct.unpack('<H', self.buf[22: 24])[0]

    def get_next_record_offset(self):
        """Get and return the offset of the free space in this page."""

        return struct.unpack('<H', self.buf[24: 26])[0]

    def get_last_end_lsn(self):
        """Get and return the log sequence number (LSN) for the last record in this page."""

        return struct.unpack('<Q', self.buf[32: 40])[0]

    def get_log_record_at_offset(self, offset, header_length=None):
        """Get, parse and return a log record header (LogRecordHeader) by its offset in this log record page."""

        if offset < self.data_offset:
            raise LogFileException('Invalid log data offset: {} < {}'.format(offset, self.data_offset))

        return LogRecordHeader(self.buf[offset:], header_length)

    def __str__(self):
        return 'LogRecordPage #{}, USA end: {}, last LSN: {} (or file offset 1.1/2.x: {}/{}), flags: {}, page count: {}, page position: {}, last end LSN: {}'.format(
            self.page_number, self.end_of_usa_offset, self.get_last_lsn(), self.get_file_offset_11(),
            self.get_file_offset_2x(), hex(self.get_flags()), self.get_page_count(), self.get_page_position(),
            self.get_last_end_lsn())


class LogRecordHeader(object):
    """This class is used to work with a log record header."""

    buf = None
    """Data of this log record (including client data)."""

    sizeof = 48
    """A size of this header (aligned to 8 bytes)."""

    def __init__(self, log_record_raw, header_length=None):
        self.buf = log_record_raw

        if header_length is not None and header_length < 48:
            raise NotImplementedError(
                'The following log record header length is not supported: {}'.format(header_length))

        if header_length is not None:
            self.sizeof = header_length

        if self.sizeof < 8 or self.sizeof % 8 != 0:
            raise LogFileException('Invalid log header length: {}'.format(self.sizeof))

        if len(self.buf) < self.sizeof:
            raise LogFileException('Truncated log record header')

    def get_this_lsn(self):
        """Get and return the log sequence number (LSN) for this record."""

        return struct.unpack('<Q', self.buf[0: 8])[0]

    def get_client_previous_lsn(self):
        """Get and return the previous log sequence number (LSN) for this client."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_client_undo_next_lsn(self):
        """Get and return the next undo log sequence number (LSN) for this client."""

        return struct.unpack('<Q', self.buf[16: 24])[0]

    def get_client_data_length(self):
        """Get and return the client data length for this record."""

        return struct.unpack('<L', self.buf[24: 28])[0]

    def get_client_id(self):
        """Get and return the client ID for this record."""

        return struct.unpack('<L', self.buf[28: 32])[0]

    def get_record_type(self):
        """Get and return the record type."""

        return struct.unpack('<L', self.buf[32: 36])[0]

    def get_transaction_id(self):
        """Get and return the transaction ID."""

        return struct.unpack('<L', self.buf[36: 40])[0]

    def get_flags(self):
        """Get and return the flags (as an integer)."""

        return struct.unpack('<H', self.buf[40: 42])[0]

    def __str__(self):
        return 'LogRecordHeader, this LSN: {}, client ID: {}, record type: {}, client data length: {}, transaction ID: {}, flags: {}'.format(
            self.get_this_lsn(), self.get_client_id(), self.get_record_type(), self.get_client_data_length(),
            self.get_transaction_id(), hex(self.get_flags()))


class LogFileParser(object):
    """This class is used to read and parse a $LogFile file."""

    file_object = None
    """A file object for a $LogFile file."""

    file_size = None
    """A size of this $LogFile file."""

    restart_page_1 = None
    """The first restart page (a RestartPage object)."""

    restart_page_2 = None
    """The second restart page (a RestartPage object)."""

    log_page_size = None
    """A size of a single log record page."""

    log_version = None
    """A tuple (major_version, minor_version) for this log file."""

    lsns = None
    """A sorted list of log sequence numbers (LSNs) seen in this file."""

    lsns_sorted = None
    """A dictionary with a sorted list of log sequence numbers (LSNs) grouped by client IDs (each client ID is a key in this dictionary)."""

    cluster_size = None
    """A number of bytes per cluster. (It will be adjusted later.)"""

    oat_version = None
    """A version of the open attribute table to use. (It will be adjusted later.)"""

    def __init__(self, file_object):
        self.file_object = file_object
        self.writable = False

        self.file_object.seek(0, 2)
        self.file_size = self.file_object.tell()

        self.file_object.seek(0)
        restart_page_1_raw = self.file_object.read(PAGE_SIZE)
        restart_page_2_raw = self.file_object.read(PAGE_SIZE)

        if len(restart_page_1_raw) != PAGE_SIZE or len(restart_page_2_raw) != PAGE_SIZE:
            raise LogFileException('Truncated log file')

        if restart_page_1_raw == restart_page_2_raw == b'\xFF' * PAGE_SIZE:
            raise EmptyLogFileException('Empty log file')

        try:
            self.restart_page_1 = RestartPage(restart_page_1_raw)
        except LogFileException:
            self.restart_page_1 = None

        try:
            self.restart_page_2 = RestartPage(restart_page_2_raw)
        except LogFileException:
            self.restart_page_2 = None

        if self.restart_page_1 is None and self.restart_page_2 is None:  # Both restart pages are corrupt.
            raise LogFileException('No valid restart pages found')

        log_page_size = None
        log_version = None

        if self.restart_page_1 is None:  # The first restart page is corrupt, use the second one.
            log_page_size = self.restart_page_2.get_log_page_size()
            log_version = (self.restart_page_2.get_major_version(), self.restart_page_2.get_minor_version())
            restart_area = self.restart_page_2.get_restart_area()

        if self.restart_page_2 is None:  # The second restart page is corrupt, use the first one.
            log_page_size = self.restart_page_1.get_log_page_size()
            log_version = (self.restart_page_1.get_major_version(), self.restart_page_1.get_minor_version())
            restart_area = self.restart_page_1.get_restart_area()

        if log_page_size is None:  # Both restart pages are valid.
            log_page_size = self.restart_page_1.get_log_page_size()
            restart_area = self.restart_page_1.get_restart_area()

            if log_page_size != self.restart_page_2.get_log_page_size():
                raise LogFileException('Invalid log page size in the second restart page')

            log_version = (self.restart_page_1.get_major_version(), self.restart_page_1.get_minor_version())
            if log_version != (self.restart_page_2.get_major_version(), self.restart_page_2.get_minor_version()):
                raise LogFileException('Invalid log version the second restart page')

        self.log_page_size = log_page_size
        self.log_version = log_version

        self.record_header_length = restart_area.get_record_header_length()
        if self.record_header_length < LogRecordHeader.sizeof:
            raise LogFileException(
                'Invalid record header length: {} < {}'.format(self.record_header_length, LogRecordHeader.sizeof))

        self.log_page_data_offset = restart_area.get_log_page_data_offset()

        self.sequence_number_bits = restart_area.get_sequence_number_bits()
        if self.sequence_number_bits >= 64 or self.sequence_number_bits < 3:
            raise LogFileException('Invalid count of sequence number bits: {}'.format(self.sequence_number_bits))

        self.cluster_size = CLUSTER_SIZE_DEFAULT  # Adjusted in the 'collect_lsns' method.
        self.oat_version = 1  # Adjusted in the 'collect_lsns' method.

    def lsn_to_offset(self, lsn):
        """Convert a log sequence number (LSN) to a file offset, return a (log_page_number, offset_in_page) tuple."""

        absolute_offset = ((lsn << self.sequence_number_bits) & 0xFFFFFFFFFFFFFFFF) >> (self.sequence_number_bits - 3)

        log_page_number = absolute_offset // self.log_page_size
        offset_in_page = absolute_offset % self.log_page_size

        return (log_page_number, offset_in_page)

    def lsn_to_seqnum(self, lsn):
        """Convert a log sequence number (LSN) to a sequence number and return it."""

        return lsn >> (64 - self.sequence_number_bits)

    def offset_to_lsn(self, log_page_number, offset_in_page, sequence_number):
        """Convert a log page number, an offset within that page, and a sequence number to a log sequence number (LSN) and return it."""

        absolute_offset = log_page_number * self.log_page_size + offset_in_page

        return (absolute_offset >> 3) | (sequence_number << (64 - self.sequence_number_bits))

    def get_log_record_page_by_number(self, log_page_number):
        """Get, parse and return a log record page (LogRecordPage) by its number (or None, if not found)."""

        self.file_object.seek(log_page_number * self.log_page_size)
        buf = self.file_object.read(self.log_page_size)

        if len(buf) != self.log_page_size:  # Truncated data.
            return

        try:
            log_page = LogRecordPage(buf, log_page_number, self.log_page_data_offset)
        except (LogFileException, NotImplementedError):
            log_page = None

        return log_page

    def get_wrapped_log_record_page_number(self):
        """Return the first log record page number to be used after the log file is wrapped."""

        if self.log_version[0] == 1 and self.log_version[1] == 1:
            start_page = 4  # 2 restart pages and 2 tail log record pages.
        elif self.log_version[0] >= 2:
            start_page = 34  # 2 restart pages and 32 fast pages.
        else:
            raise NotImplementedError('Unknown location of the "infinite" area')

        return start_page

    def collect_lsns(self):
        """Collect valid log sequence numbers (LSNs) found in this log file."""

        self.lsns = set()

        if self.restart_page_1 is not None:
            lsn = self.restart_page_1.get_chkdsk_lsn()
            self.lsns.add(lsn)

            lsn = self.restart_page_1.get_restart_area().get_current_lsn()
            self.lsns.add(lsn)

            for client in self.restart_page_1.get_restart_area().clients():
                lsn = client.get_oldest_lsn()
                self.lsns.add(lsn)

                lsn = client.get_client_restart_lsn()
                self.lsns.add(lsn)

        if self.restart_page_2 is not None:
            lsn = self.restart_page_2.get_chkdsk_lsn()
            self.lsns.add(lsn)

            lsn = self.restart_page_2.get_restart_area().get_current_lsn()
            self.lsns.add(lsn)

            for client in self.restart_page_2.get_restart_area().clients():
                lsn = client.get_oldest_lsn()
                self.lsns.add(lsn)

                lsn = client.get_client_restart_lsn()
                self.lsns.add(lsn)

        for log_record_page in self.log_record_pages(1):
            lsn = log_record_page.get_last_lsn()
            self.lsns.add(lsn)

            lsn = log_record_page.get_last_end_lsn()
            self.lsns.add(lsn)

        # Validate every LSN found, collect new LSNs.
        new_lsns = set()
        for lsn in self.lsns.copy():
            log_page_number, offset = self.lsn_to_offset(lsn)

            if log_page_number == 0:
                self.lsns.remove(lsn)
                continue

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                self.lsns.remove(lsn)
                continue

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset, self.record_header_length)
            except LogFileException:
                self.lsns.remove(lsn)
                continue

            if lsn != log_record_header.get_this_lsn():
                self.lsns.remove(lsn)

            new_lsn = log_record_header.get_client_previous_lsn()
            new_lsns.add(new_lsn)

            new_lsn = log_record_header.get_client_undo_next_lsn()
            new_lsns.add(new_lsn)

        # Validate these new LSNs.
        for lsn in new_lsns.copy():
            log_page_number, offset = self.lsn_to_offset(lsn)

            if log_page_number == 0:
                new_lsns.remove(lsn)
                continue

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                new_lsns.remove(lsn)
                continue

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset, self.record_header_length)
            except LogFileException:
                new_lsns.remove(lsn)
                continue

            if lsn != log_record_header.get_this_lsn():
                new_lsns.remove(lsn)

        # Extend the list of LSNs found using new data.
        self.lsns.update(new_lsns)

        # Now, walk through all LSNs found before and locate new LSNs.

        def find_next_lsn(current_lsn):
            """Given a current LSN, find, validate and return the next LSN. If there is none or something is invalid, return None."""

            log_page_number, offset_in_page = self.lsn_to_offset(current_lsn)
            if log_page_number == 0:
                return

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                return

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset_in_page, self.record_header_length)
            except LogFileException:
                return

            client_data_length = log_record_header.get_client_data_length()
            if client_data_length < 8 or client_data_length % 8 != 0:
                return

            seqnum_expected = self.lsn_to_seqnum(lsn)

            if client_data_length <= self.log_page_size - offset_in_page - self.record_header_length:  # Client data is in the current log record page only.
                # Try the current log record page.
                candidate_offset_in_page = offset_in_page + self.record_header_length + client_data_length
                candidate_lsn = self.offset_to_lsn(log_page_number, candidate_offset_in_page, seqnum_expected)

                try:
                    candidate_log_record_header = log_record_page.get_log_record_at_offset(candidate_offset_in_page,
                                                                                           self.record_header_length)
                except LogFileException:
                    pass
                else:
                    if candidate_log_record_header.get_this_lsn() == candidate_lsn:
                        return candidate_lsn

                # Try the next log record page.
                candidate_log_page_number = log_page_number + 1
                candidate_offset_in_page = self.log_page_data_offset
                candidate_lsn = self.offset_to_lsn(candidate_log_page_number, candidate_offset_in_page, seqnum_expected)

                # Handle the wrap.
                if candidate_log_page_number * self.log_page_size >= self.file_size:
                    candidate_log_page_number = self.get_wrapped_log_record_page_number()
                    seqnum_expected += 1
                    candidate_lsn = self.offset_to_lsn(candidate_log_page_number, candidate_offset_in_page,
                                                       seqnum_expected)

                candidate_log_record_page = self.get_log_record_page_by_number(candidate_log_page_number)
                if candidate_log_record_page is not None:
                    try:
                        candidate_log_record_header = candidate_log_record_page.get_log_record_at_offset(
                            candidate_offset_in_page, self.record_header_length)
                    except LogFileException:
                        pass
                    else:
                        if candidate_log_record_header.get_this_lsn() == candidate_lsn:
                            return candidate_lsn

            else:  # Client data is in two or more log record pages.
                client_data_length_left = client_data_length - (
                        self.log_page_size - offset_in_page - self.record_header_length)  # Client data length in the next pages (without the first one).

                pages_to_skip = 1 + client_data_length_left // (
                        self.log_page_size - self.log_page_data_offset)  # The number of pages to skip (including the first one with client data).

                candidate_log_page_number = log_page_number + pages_to_skip

                candidate_offset_in_page = self.log_page_data_offset + client_data_length_left - (pages_to_skip - 1) * (
                        self.log_page_size - self.log_page_data_offset)
                candidate_lsn = self.offset_to_lsn(candidate_log_page_number, candidate_offset_in_page, seqnum_expected)

                # Handle the wrap.
                if candidate_log_page_number * self.log_page_size >= self.file_size:
                    pages_left_in_file = (self.file_size // self.log_page_size) - (
                            log_page_number + 1)  # Pages left after the first one with client data.

                    candidate_log_page_number = self.get_wrapped_log_record_page_number() + (
                            pages_to_skip - 1) - pages_left_in_file
                    seqnum_expected += 1
                    candidate_lsn = self.offset_to_lsn(candidate_log_page_number, candidate_offset_in_page,
                                                       seqnum_expected)

                candidate_log_record_page = self.get_log_record_page_by_number(candidate_log_page_number)
                if candidate_log_record_page is not None:
                    try:
                        candidate_log_record_header = candidate_log_record_page.get_log_record_at_offset(
                            candidate_offset_in_page, self.record_header_length)
                    except LogFileException:
                        pass
                    else:
                        if candidate_log_record_header.get_this_lsn() == candidate_lsn:
                            return candidate_lsn

                # Try the next log record page.
                candidate_log_page_number += 1
                candidate_offset_in_page = self.log_page_data_offset
                candidate_lsn = self.offset_to_lsn(candidate_log_page_number, candidate_offset_in_page, seqnum_expected)

                candidate_log_record_page = self.get_log_record_page_by_number(candidate_log_page_number)
                if candidate_log_record_page is not None:
                    try:
                        candidate_log_record_header = candidate_log_record_page.get_log_record_at_offset(
                            candidate_offset_in_page, self.record_header_length)
                    except LogFileException:
                        pass
                    else:
                        if candidate_log_record_header.get_this_lsn() == candidate_lsn:
                            return candidate_lsn

        def find_previous_lsn(current_lsn):
            """Give a current LSN, find, validate and return the previous LSN for the same client. If there is none or something is invalid, return None."""

            log_page_number, offset_in_page = self.lsn_to_offset(current_lsn)
            if log_page_number == 0:
                return

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                return

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset_in_page, self.record_header_length)
            except LogFileException:
                return

            client_previous_lsn = log_record_header.get_client_previous_lsn()
            candidate_log_page_number, candidate_offset_in_page = self.lsn_to_offset(client_previous_lsn)

            if candidate_log_page_number == 0:
                return

            candidate_log_record_page = self.get_log_record_page_by_number(candidate_log_page_number)
            if candidate_log_record_page is not None:
                try:
                    candidate_log_record_header = candidate_log_record_page.get_log_record_at_offset(
                        candidate_offset_in_page, self.record_header_length)
                except LogFileException:
                    pass
                else:
                    if candidate_log_record_header.get_this_lsn() == client_previous_lsn:
                        return client_previous_lsn

        def find_undo_next_lsn(current_lsn):
            """Give a current LSN, find, validate and return the undo next LSN for the same client. If there is none or something is invalid, return None."""

            log_page_number, offset_in_page = self.lsn_to_offset(current_lsn)
            if log_page_number == 0:
                return

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                return

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset_in_page, self.record_header_length)
            except LogFileException:
                return

            client_undo_next_lsn = log_record_header.get_client_undo_next_lsn()
            candidate_log_page_number, candidate_offset_in_page = self.lsn_to_offset(client_undo_next_lsn)

            if candidate_log_page_number == 0:
                return

            candidate_log_record_page = self.get_log_record_page_by_number(candidate_log_page_number)
            if candidate_log_record_page is not None:
                try:
                    candidate_log_record_header = candidate_log_record_page.get_log_record_at_offset(
                        candidate_offset_in_page, self.record_header_length)
                except LogFileException:
                    pass
                else:
                    if candidate_log_record_header.get_this_lsn() == client_undo_next_lsn:
                        return client_undo_next_lsn

        # Find new LSNs by walking over client data.
        for lsn in self.lsns.copy():
            current_lsn = lsn

            while True:
                current_lsn = find_next_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

        # Find new LSNs by looking for previous LSNs recorded for the same client.
        for lsn in self.lsns.copy():
            current_lsn = lsn

            while True:
                current_lsn = find_previous_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

        # Find new LSNs by looking for undo next LSNs for the same client.
        for lsn in self.lsns.copy():
            current_lsn = lsn

            while True:
                current_lsn = find_undo_next_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

        # Find new LSNs by looking in NTFS restart areas.
        new_lsns = set()

        for lsn in self.lsns.copy():
            client_data = self.get_client_data(lsn)
            if client_data.record_type == LfsClientRestart:
                try:
                    ntfs_restart_area = client_data.data_decoded()
                except ClientException:
                    continue

                # Adjust the cluster size.
                cluster_size = ntfs_restart_area.get_bytes_per_cluster()
                if cluster_size is not None and cluster_size > 0 and cluster_size % 512 == 0:
                    self.cluster_size = cluster_size

                # Adjust the version-specific configuration.
                self.use_dumps = True
                major_version = ntfs_restart_area.get_major_version()
                if major_version >= 1:
                    if major_version >= 2:
                        self.use_dumps = False

                    self.oat_version = 1
                else:
                    self.oat_version = 0

                new_lsns.add(ntfs_restart_area.get_start_of_checkpoint_lsn())
                new_lsns.add(ntfs_restart_area.get_open_attribute_table_lsn())
                new_lsns.add(ntfs_restart_area.get_attribute_names_lsn())
                new_lsns.add(ntfs_restart_area.get_dirty_page_table_lsn())
                new_lsns.add(ntfs_restart_area.get_transaction_table_lsn())

        for lsn in new_lsns:
            # Validate a new LSN.
            log_page_number, offset = self.lsn_to_offset(lsn)
            if log_page_number == 0:
                continue

            log_record_page = self.get_log_record_page_by_number(log_page_number)
            if log_record_page is None:
                continue

            try:
                log_record_header = log_record_page.get_log_record_at_offset(offset, self.record_header_length)
            except LogFileException:
                continue

            if lsn == log_record_header.get_this_lsn():
                self.lsns.add(lsn)
            else:
                continue

            # Search for new LSNs.
            current_lsn = lsn
            while True:
                current_lsn = find_next_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

            current_lsn = lsn
            while True:
                current_lsn = find_previous_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

            current_lsn = lsn
            while True:
                current_lsn = find_undo_next_lsn(current_lsn)

                if current_lsn is None or current_lsn in self.lsns:  # Loop until a known LSN is encountered or something invalid is found.
                    break
                else:
                    self.lsns.add(current_lsn)

        self.lsns = sorted(self.lsns)  # Finally, store a sorted list of LSNs.

    def sort_lsns(self):
        """Group log sequence numbers (LSNs) by their client IDs."""

        self.lsns_sorted = dict()

        for current_lsn in self.lsns:
            log_page_number, offset_in_page = self.lsn_to_offset(current_lsn)
            log_record_page = self.get_log_record_page_by_number(log_page_number)
            log_record_header = log_record_page.get_log_record_at_offset(offset_in_page, self.record_header_length)

            client_id = log_record_header.get_client_id()
            if client_id not in self.lsns_sorted.keys():
                self.lsns_sorted[client_id] = [current_lsn]
            else:
                self.lsns_sorted[client_id].append(current_lsn)

        for client_id in self.lsns_sorted.keys():
            self.lsns_sorted[client_id] = sorted(self.lsns_sorted[client_id])

    def prepare_for_writes(self):
        """Make a writable file object in memory (which is a copy of the original one) and use it for further operations."""

        if self.writable:
            return

        self.file_object.seek(0)

        new_file_object = io.BytesIO()
        shutil.copyfileobj(self.file_object, new_file_object)

        self.old_file_object = self.file_object
        self.file_object = new_file_object
        self.writable = True

    def discard_changes(self):
        """Discard all changes made to a writable file object (in memory) and use the original file object."""

        if not self.writable:
            return

        self.file_object.close()
        self.file_object = self.old_file_object
        self.writable = False

    def apply_tail_page(self):
        """Apply a tail log record page to the log file. The log file version should be 1.1. The file object should be writable."""

        if self.log_version != (1, 1):
            raise RuntimeError('The log file version is not 1.1')

        tail_page_1 = self.get_log_record_page_by_number(2)
        tail_page_2 = self.get_log_record_page_by_number(3)

        if tail_page_1 is None and tail_page_2 is None:  # Nothing to do, both tail log record pages are invalid.
            return

        single_tail_page = None

        if tail_page_1 is None and tail_page_2 is not None:  # Only the second tail log record page is present.
            single_tail_page = tail_page_2
        elif tail_page_2 is None and tail_page_1 is not None:  # Only the first tail log record page is present.
            single_tail_page = tail_page_1
        elif tail_page_1 is not None and tail_page_2 is not None:  # Both tail log record pages are valid, choose the latest one.
            if tail_page_1.get_last_end_lsn() > tail_page_2.get_last_end_lsn():
                single_tail_page = tail_page_1
            else:
                single_tail_page = tail_page_2

        if single_tail_page is not None:  # A proper tail log record page was found.
            file_offset = single_tail_page.get_file_offset_11()

            self.file_object.seek(file_offset)
            self.file_object.write(single_tail_page.buf_protected)

    def apply_fast_pages(self):
        """Apply fast log record pages to the log file. The log file version should be 2.0 or higher. The file object should be writable."""

        if self.log_version[0] < 2:
            raise RuntimeError('The log file version is not 2.x (or higher)')

        # Locate the highest LSN present in the "infinite" area.
        highest_lsn = 0
        for log_record_page in self.log_record_pages(0):
            current_lsn = log_record_page.get_last_lsn()
            if current_lsn > highest_lsn:
                highest_lsn = current_lsn

        fast_pages = []

        i = 2
        while i < 34:
            fast_page = self.get_log_record_page_by_number(i)
            if fast_page is not None and fast_page.get_last_lsn() > highest_lsn:
                fast_pages.append(fast_page)

            i += 1

        fast_pages.sort(key=lambda
            x: x.get_last_lsn())  # Sort the fast pages according to the log sequence numbers they have (fast pages with more recent log data should be written at the end).

        for fast_page in fast_pages:
            file_offset = fast_page.get_file_offset_2x()

            self.file_object.seek(file_offset)
            self.file_object.write(fast_page.buf_protected)

    def log_record_pages(self, mode):
        """This method yields each log record page (LogRecordPage).
        When 'mode' is 0, yield log record pages from the "infinite" (circular) area only.
        When 'mode' is 1, yield each log record page from the whole file (not just from the "infinite" area).
        """

        if mode not in [0, 1]:
            raise ValueError('Invalid mode of operation: {}'.format(mode))

        if mode == 0:
            start_page = self.get_wrapped_log_record_page_number()
        elif mode == 1:
            start_page = 2  # 2 restart pages.

        i = start_page
        while i < self.file_size // self.log_page_size:
            log_page = self.get_log_record_page_by_number(i)

            if log_page is not None:
                yield log_page

            i += 1

    def get_client_data(self, lsn):
        """Get, parse and return client data (ClientData) for a given log sequence number (LSN)."""

        log_page_number, offset_in_page = self.lsn_to_offset(lsn)

        log_record_page = self.get_log_record_page_by_number(log_page_number)
        log_record_header = log_record_page.get_log_record_at_offset(offset_in_page, self.record_header_length)

        client_data_length = log_record_header.get_client_data_length()

        if client_data_length <= self.log_page_size - offset_in_page - self.record_header_length:  # Client data is in the current log record page only.
            client_buf = log_record_header.buf[
                         self.record_header_length: self.record_header_length + client_data_length]
            client_data = ClientData(client_buf, log_record_header.get_record_type(), lsn,
                                     log_record_header.get_transaction_id(), self.cluster_size)

            return client_data

        else:  # Client data is in two or more log record pages.
            client_buf = log_record_header.buf[self.record_header_length:]
            client_data_length_left = client_data_length - len(client_buf)

            while client_data_length_left > 0:
                log_page_number += 1

                # Handle the wrap.
                if log_page_number * self.log_page_size >= self.file_size:
                    log_page_number = self.get_wrapped_log_record_page_number()

                log_record_page = self.get_log_record_page_by_number(log_page_number)
                if log_record_page is None:  # This page is missing in the current state of the log file.
                    break

                if client_data_length_left >= self.log_page_size - self.log_page_data_offset:
                    client_buf_more_data = log_record_page.buf[self.log_page_data_offset:]
                else:
                    client_buf_more_data = log_record_page.buf[
                                           self.log_page_data_offset: self.log_page_data_offset + client_data_length_left]

                client_buf += client_buf_more_data

                client_data_length_left -= len(client_buf_more_data)

            client_data = ClientData(client_buf, log_record_header.get_record_type(), lsn,
                                     log_record_header.get_transaction_id(), self.cluster_size)

            return client_data

    def parse_ntfs_records(self, recover_log_data=True):
        """This method yields NTFS client data (NTFSRestartArea and NTFSLogRecord) from each log record found in the log file.
        If 'recover_log_data' is True, apply a tail log record page or fast log record pages before parsing the log data.
        """

        if recover_log_data:
            self.prepare_for_writes()

            if self.log_version == (1, 1):
                self.apply_tail_page()
            elif self.log_version[0] >= 2:
                self.apply_fast_pages()

        self.collect_lsns()
        self.sort_lsns()

        for client_id in self.lsns_sorted.keys():
            latest_oat_dump = None
            old_oat_dump = None
            latest_an_dump = None
            old_an_dump = None

            oat_dump_count = 0
            an_dump_count = 0

            oat = dict()

            for lsn in self.lsns_sorted[client_id]:
                client_data = self.get_client_data(lsn)
                try:
                    client_data_decoded = client_data.data_decoded()
                except NotImplementedError:
                    pass
                else:
                    if type(client_data_decoded) is NTFSLogRecord:
                        redo_op = client_data_decoded.get_redo_operation()
                        if redo_op == OpenNonresidentAttribute:
                            attribute_name = client_data_decoded.get_undo_data().decode('utf-16le', errors='replace')

                            oat_index = client_data_decoded.get_target_attribute()
                            oat_entry_data = client_data_decoded.get_redo_data()

                            try:
                                if self.oat_version >= 1:
                                    oat_entry = OpenAttributeEntry1(oat_entry_data)
                                else:
                                    oat_entry = OpenAttributeEntry0(oat_entry_data)
                            except ClientException:
                                # Something is wrong, clear the open attribute table.
                                oat = dict()
                            else:
                                oat[oat_index] = (oat_entry.get_file_reference(), attribute_name)
                        elif self.use_dumps and redo_op == OpenAttributeTableDump:
                            old_oat_dump = latest_oat_dump
                            latest_oat_dump = OpenAttributeTableDumpParser(client_data_decoded.get_redo_data(),
                                                                           self.oat_version)
                            oat_dump_count += 1
                        elif self.use_dumps and redo_op == AttributeNamesDump:
                            old_an_dump = latest_an_dump
                            latest_an_dump = AttributeNamesDumpParser(client_data_decoded.get_redo_data())
                            an_dump_count += 1

                        client_data_decoded.oat = oat.copy()

                        if self.use_dumps and oat_dump_count > 0 and oat_dump_count == an_dump_count:
                            client_data_decoded.oat_dump = latest_oat_dump
                            client_data_decoded.an_dump = latest_an_dump
                        elif self.use_dumps and oat_dump_count > 0 and oat_dump_count != an_dump_count:
                            client_data_decoded.oat_dump = old_oat_dump
                            client_data_decoded.an_dump = old_an_dump
                        else:
                            client_data_decoded.oat_dump = None
                            client_data_decoded.an_dump = None

                    yield client_data_decoded

        if recover_log_data:
            self.discard_changes()

    def __str__(self):
        return 'LogFileParser, file size: {}'.format(self.file_size)


class ClientData(object):
    """This class is used to work with client data."""

    buf = None
    """Client data."""

    record_type = None
    """A record type of this client data."""

    lsn = None
    """A log sequence number (LSN) for this client data."""

    cluster_size = None
    """A number of bytes per cluster."""

    transaction_id = None
    """A transaction ID."""

    def __init__(self, client_data_raw, record_type, log_sequence_number, transaction_id, cluster_size):
        self.buf = client_data_raw
        self.record_type = record_type
        self.lsn = log_sequence_number
        self.transaction_id = transaction_id
        self.cluster_size = cluster_size

    def data_decoded(self):
        """Parse and return decoded data (NTFSRestartArea or NTFSLogRecord). Only the NTFS client is supported."""

        if self.record_type == LfsClientRestart:
            return NTFSRestartArea(self.buf, self.lsn)
        elif self.record_type == LfsClientRecord:
            return NTFSLogRecord(self.buf, self.lsn, self.transaction_id, self.cluster_size)
        else:
            raise NotImplementedError('The following record type is not supported: {}'.format(self.record_type))

    def __str__(self):
        return 'ClientData, length: {}, record type: {}, LSN: {}'.format(len(self.buf), self.record_type, self.lsn)


class NTFSRestartArea(object):
    """This class is used work with an NTFS restart area."""

    buf = None
    """Data of this NTFS restart area."""

    lsn = None
    """A log sequence number (LSN) of an NTFS restart area."""

    def __init__(self, restart_area_raw, lsn):
        self.buf = restart_area_raw
        self.lsn = lsn

        if len(self.buf) < 64:
            raise ClientException('Invalid restart area length: {}'.format(len(self.buf)))

    def get_major_version(self):
        """Get and return the major version of a client."""

        return struct.unpack('<L', self.buf[: 4])[0]

    def get_minor_version(self):
        """Get and return the minor version of a client."""

        return struct.unpack('<L', self.buf[4: 8])[0]

    def get_start_of_checkpoint_lsn(self):
        """Get and return the log sequence number (LSN) of a checkpoint record."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_open_attribute_table_lsn(self):
        """Get and return the log sequence number (LSN) of an open attribute table."""

        return struct.unpack('<Q', self.buf[16: 24])[0]

    def get_attribute_names_lsn(self):
        """Get and return the log sequence number (LSN) of attribute names."""

        return struct.unpack('<Q', self.buf[24: 32])[0]

    def get_dirty_page_table_lsn(self):
        """Get and return the log sequence number (LSN) of a dirty page table."""

        return struct.unpack('<Q', self.buf[32: 40])[0]

    def get_transaction_table_lsn(self):
        """Get and return the log sequence number (LSN) of a transaction table."""

        return struct.unpack('<Q', self.buf[40: 48])[0]

    def get_open_attribute_table_length(self):
        """Get and return the length of an open attribute table."""

        return struct.unpack('<L', self.buf[48: 52])[0]

    def get_attribute_names_length(self):
        """Get and return the length of attribute names."""

        return struct.unpack('<L', self.buf[52: 56])[0]

    def get_dirty_page_table_length(self):
        """Get and return the length of a dirty page table."""

        return struct.unpack('<L', self.buf[56: 60])[0]

    def get_transaction_table_length(self):
        """Get and return the length of a transaction table."""

        return struct.unpack('<L', self.buf[60: 64])[0]

    def get_usn_journal_restart_offset(self):
        """Get and return the absolute offset used as a starting point for checking update sequence number (USN) change journal records."""

        if len(self.buf) >= 72:
            return struct.unpack('<q', self.buf[64: 72])[0]

    def get_last_lsn(self):
        """Get and return the last value (as of the previous volume restart) of the "current" log sequence number (LSN)."""

        if len(self.buf) >= 80:
            return struct.unpack('<Q', self.buf[72: 80])[0]

    def get_bytes_per_cluster(self):
        """Get and return the number of bytes per cluster."""

        if len(self.buf) >= 84:
            return struct.unpack('<L', self.buf[80: 84])[0]

    def get_usn_journal_reference(self):
        """Get and return the file record segment (FRS) reference for the update sequence number (USN) change journal."""

        if len(self.buf) >= 96:
            return struct.unpack('<Q', self.buf[88: 96])[0]

    def get_usn_base(self):
        """Get and return the base offset for the update sequence number (USN) change journal.
        Since the $J data stream of this journal can be very large, this value is used to rebase offsets for high-level API functions (making them relative from the base offset).
        """

        if len(self.buf) >= 104:
            return struct.unpack('<q', self.buf[96: 104])[0]

    def get_oldest_lsn(self):
        """Get and return the oldest log sequence number (LSN) required by this client (as of the checkpoint)."""

        if len(self.buf) >= 112:
            return struct.unpack('<Q', self.buf[104: 112])[0]

    def __str__(self):
        return 'NTFSRestartArea (LSN: {}), version: {}.{}, start of checkpoint LSN: {}'.format(self.lsn,
                                                                                               self.get_major_version(),
                                                                                               self.get_minor_version(),
                                                                                               self.get_start_of_checkpoint_lsn())


class NTFSLogRecord(object):
    """This class is used work with an NTFS log record."""

    buf = None
    """Data of this NTFS log record."""

    lsn = None
    """A log sequence number (LSN) of this NTFS log record."""

    oat = None
    """An open attribute table (it is set externally in the 'parse_ntfs_records' method of the LogFileParser class)."""

    oat_dump = None
    """An open attribute table dump (as raw bytes; it is set externally in the 'parse_ntfs_records' method of the LogFileParser class)."""

    an_dump = None
    """An attribute names dump (as raw bytes; it is set externally in the 'parse_ntfs_records' method of the LogFileParser class)."""

    transaction_id = None
    """A transaction ID."""

    def __init__(self, log_record_raw, lsn, transaction_id, cluster_size):
        self.buf = log_record_raw
        self.lsn = lsn
        self.transaction_id = transaction_id

        self.cluster_size = cluster_size

        if len(self.buf) < 32:
            raise ClientException('Invalid log record length: {}'.format(len(self.buf)))

    def get_redo_operation(self):
        """Get and return the redo operation code (as an integer)."""

        return struct.unpack('<H', self.buf[0: 2])[0]

    def get_undo_operation(self):
        """Get and return the undo operation code (as an integer)."""

        return struct.unpack('<H', self.buf[2: 4])[0]

    def get_redo_offset(self):
        """Get and return the offset of an undo record."""

        offset = struct.unpack('<H', self.buf[4: 6])[0]
        if 0 < offset < 32:
            raise ClientException('Invalid redo offset: {}'.format(offset))

        return offset

    def get_redo_length(self):
        """Get and return the length of an undo record."""

        return struct.unpack('<H', self.buf[6: 8])[0]

    def get_undo_offset(self):
        """Get and return the offset of an undo record."""

        offset = struct.unpack('<H', self.buf[8: 10])[0]
        if 0 < offset < 32:
            raise ClientException('Invalid undo offset: {}'.format(offset))

        return offset

    def get_undo_length(self):
        """Get and return the length of an undo record."""

        return struct.unpack('<H', self.buf[10: 12])[0]

    def get_target_attribute(self):
        """Get and return the open attribute table index for this update."""

        return struct.unpack('<H', self.buf[12: 14])[0]

    def get_lcns_to_follow(self):
        """Get and return the number of logical cluster numbers (LCNs) in use at the end of the header."""

        return struct.unpack('<H', self.buf[14: 16])[0]

    def get_record_offset(self):
        """Get and return the record offset."""

        return struct.unpack('<H', self.buf[16: 18])[0]

    def get_attribute_offset(self):
        """Get and return the attribute offset."""

        return struct.unpack('<H', self.buf[18: 20])[0]

    def get_cluster_block_offset(self):
        """Get and return the cluster block offset (this is a number of 512-byte blocks)."""

        return struct.unpack('<H', self.buf[20: 22])[0]

    def get_target_block_size(self):
        """For operations on the $MFT file, this field is set to the size (in bytes) of the file record segment (FRS) divided by 512.
        If the attribute is $INDEX_ALLOCATION, this field is set to the size (in bytes) of the index record divided by 512.
        In other cases, this field is set to 0.
        """

        return struct.unpack('<H', self.buf[22: 24])[0]

    def get_target_vcn(self):
        """Get and return the target virtual cluster number (VCN)."""

        return struct.unpack('<q', self.buf[24: 32])[0]

    def get_lcns_for_page(self):
        """Get and return the array of logical cluster numbers (LCNs) as a list of integers."""

        lcns = []

        i = 0
        while i < self.get_lcns_to_follow():
            lcn_buf = self.buf[32 + i * 8: 40 + i * 8]
            if len(lcn_buf) != 8:
                raise ClientException('Invalid (truncated) log record length: {}'.format(len(self.buf)))

            lcn = struct.unpack('<q', lcn_buf)[0]

            lcns.append(lcn)

            i += 1

        return lcns

    def get_redo_data(self):
        """Get and return redo data."""

        redo_length = self.get_redo_length()
        if redo_length == 0:
            return b''

        redo_offset = self.get_redo_offset()
        return self.buf[redo_offset: redo_offset + redo_length]

    def get_undo_data(self):
        """Get and return undo data."""

        undo_length = self.get_undo_length()
        if undo_length == 0:
            return b''

        undo_offset = self.get_undo_offset()
        return self.buf[undo_offset: undo_offset + undo_length]

    def calculate_mft_target_number(self):
        """Calculate and return the number of a target file record segment (or None, if not applicable).
        Note: this is for resident updates only.
        """

        if self.get_redo_operation() in LOGGED_RESIDENT_UPDATES or self.get_undo_operation() in LOGGED_RESIDENT_UPDATES:
            target_vcn = self.get_target_vcn()
            cluster_block_offset = self.get_cluster_block_offset()

            target_block_size = self.get_target_block_size()
            if target_block_size > 0:
                frs_size = target_block_size * 512
            else:
                frs_size = FILE_RECORD_SEGMENT_SIZE_DEFAULT

            return (target_vcn * self.cluster_size + cluster_block_offset * 512) // frs_size

    def calculate_offset_in_target(self):
        """Calculate and return the offset within target data (or None, if not applicable)."""

        if self.get_redo_operation() in LOGGED_RESIDENT_UPDATES or self.get_undo_operation() in LOGGED_RESIDENT_UPDATES:
            return self.get_attribute_offset() + self.get_record_offset()

        elif self.get_redo_operation() in LOGGED_NONRESIDENT_UPDATES or self.get_undo_operation() in LOGGED_NONRESIDENT_UPDATES:
            return self.get_target_vcn() * self.cluster_size + self.get_cluster_block_offset() * 512 + self.get_attribute_offset() + self.get_record_offset()

    def calculate_mft_target_reference_and_name(self):
        """Calculate and return the reference to a target file record segment (FRS) and a target attribute name as a tuple (or None, if not applicable or not found).
        Note: this is for nonresident updates present in an open attribute table.
        """

        if self.oat is None:
            return

        if self.get_redo_operation() in LOGGED_NONRESIDENT_UPDATES or self.get_undo_operation() in LOGGED_NONRESIDENT_UPDATES:
            target_attribute = self.get_target_attribute()
            if target_attribute in self.oat.keys():
                return self.oat[target_attribute]

            # If not found, try the table dumps.
            if self.oat_dump is not None and self.an_dump is not None:
                try:
                    target_file_reference = self.oat_dump.find_file_reference_by_index(target_attribute)
                except ClientException:
                    pass
                else:
                    target_attribute_name = self.an_dump.find_name_by_index(target_attribute)
                    return (target_file_reference, target_attribute_name)

    def __str__(self):
        if self.get_redo_offset() > 0:
            redo_length = self.get_redo_length()
        else:
            redo_length = 0

        if self.get_undo_offset() > 0:
            undo_length = self.get_undo_length()
        else:
            undo_length = 0

        if redo_length > 0 and redo_length == undo_length and self.get_redo_offset() == self.get_undo_offset():
            redo_undo_status = '=='
        else:
            redo_undo_status = '!='

        frs_number = self.calculate_mft_target_number()
        if frs_number is not None:
            target_spec = '{} (FRS number)'.format(frs_number)
        else:
            frs_reference_and_attribute_name = self.calculate_mft_target_reference_and_name()
            if frs_reference_and_attribute_name is not None:
                frs_reference, attribute_name = frs_reference_and_attribute_name
                target_spec = '{}/{} (FRS reference / attribute name)'.format(frs_reference, attribute_name)
            else:
                target_spec = 'unknown'

        target_offset = self.calculate_offset_in_target()
        if target_offset is None:
            target_offset = 'unknown'

        return 'NTFSLogRecord (LSN: {}), redo operation: {} (length: {}), undo operation: {} (length: {}), redo {} undo, target: {}, offset in target: {}'.format(
            self.lsn, ResolveNTFSOperation(self.get_redo_operation()), redo_length,
            ResolveNTFSOperation(self.get_undo_operation()), undo_length, redo_undo_status, target_spec, target_offset)


class OpenAttributeEntry1(object):
    """This class is used to work with the OPEN_ATTRIBUTE_ENTRY structure (version: 1.0+)."""

    buf = None
    """Data of this OPEN_ATTRIBUTE_ENTRY structure."""

    def __init__(self, open_attribute_entry_raw):
        self.buf = open_attribute_entry_raw

        if len(self.buf) < 40:
            raise ClientException('Invalid open attribute entry length: {}'.format(len(self.buf)))

    def get_allocated_or_next_free(self):
        """Get the free link (this value is 0xFFFFFFFF if this entry is allocated)."""

        return struct.unpack('<L', self.buf[0: 4])[0]

    def get_bytes_per_index_buffer(self):
        """Get and return the number of bytes per index buffer."""

        return struct.unpack('<L', self.buf[4: 8])[0]

    def get_attribute_type_code(self):
        """Get and return the attribute type code."""

        return struct.unpack('<L', self.buf[8: 12])[0]

    # At least one source states that there is a field between these two fields, but there is none.

    def get_file_reference(self):
        """Get and return the file reference."""

        return struct.unpack('<Q', self.buf[16: 24])[0]

    def get_lsn_of_open_record(self):
        """Get and return the log sequence number (LSN) of an open log record (OpenNonresidentAttribute)."""

        return struct.unpack('<Q', self.buf[24: 32])[0]


# No field for the pointer is supported (because it is valid for a running machine only).


class OpenAttributeEntry0(object):
    """This class is used to work with the OPEN_ATTRIBUTE_ENTRY structure (version: 0.0)."""

    buf = None
    """Data of this OPEN_ATTRIBUTE_ENTRY structure."""

    def __init__(self, open_attribute_entry_raw):
        self.buf = open_attribute_entry_raw

        if len(self.buf) < 44:
            raise ClientException('Invalid open attribute entry length: {}'.format(len(self.buf)))

    def get_allocated_or_next_free(self):
        """Get the free link (this value is 0xFFFFFFFF if this entry is allocated)."""

        return struct.unpack('<L', self.buf[0: 4])[0]

    def get_table_index(self):
        """Get and return the table index (offset) of an original (version: 1.0+) entry (in a table in memory)."""

        return struct.unpack('<L', self.buf[4: 8])[0]

    def get_file_reference(self):
        """Get and return the file reference."""

        return struct.unpack('<Q', self.buf[8: 16])[0]

    def get_lsn_of_open_record(self):
        """Get and return the log sequence number (LSN) of an open log record (OpenNonresidentAttribute)."""

        return struct.unpack('<Q', self.buf[16: 24])[0]

    def get_attribute_type_code(self):
        """Get and return the attribute type code."""

        return struct.unpack('<L', self.buf[28: 32])[0]

    def get_bytes_per_index_buffer(self):
        """Get and return the number of bytes per index buffer."""

        return struct.unpack('<L', self.buf[40: 44])[0]


class AttributeNameEntry(object):
    """This class is used to work with the ATTRIBUTE_NAME_ENTRY structure."""

    buf = None
    """Data of this ATTRIBUTE_NAME_ENTRY structure."""

    def __init__(self, attribute_name_entry_raw):
        self.buf = attribute_name_entry_raw

        if len(self.buf) < 8:
            raise ClientException('Invalid attribute name entry length: {}'.format(len(self.buf)))

    def get_index(self):
        """Get and return the table index (offset) of a corresponding entry in the open attribute table."""

        return struct.unpack('<H', self.buf[0: 2])[0]

    def get_name_length(self):
        """Get and return the name length."""

        return struct.unpack('<H', self.buf[2: 4])[0]

    def get_name(self):
        """Get and return the name."""

        name_buf = self.buf[4: 4 + self.get_name_length()]
        name = name_buf.decode('utf-16le', errors='replace')

        return name

    def calculate_sizeof(self):
        """Calculate and return the size of this entry."""

        return 4 + self.get_name_length() + 2  # Include the terminating null character.


class AttributeNamesDumpParser(object):
    """This class is used to work with the array of ATTRIBUTE_NAME_ENTRY structures."""

    buf = None
    """Data of this array."""

    def __init__(self, attribute_names_dump_raw):
        self.buf = attribute_names_dump_raw

    def find_name_by_index(self, index):
        """Find and return the name by its index (or None, if not found)."""

        pos = 0
        while pos < len(self.buf):
            attribute_name_entry_buf = self.buf[pos:]

            try:
                attribute_name_entry = AttributeNameEntry(attribute_name_entry_buf)
            except ClientException:
                break

            attribute_name_entry_index = attribute_name_entry.get_index()
            if attribute_name_entry_index == index:
                return attribute_name_entry.get_name()

            pos += attribute_name_entry.calculate_sizeof()


class OpenAttributeTableDumpParser(object):
    """This class is used to work with the array of OPEN_ATTRIBUTE_ENTRY structures."""

    buf = None
    """Data of this array."""

    oat_class = None
    """Either OpenAttributeEntry0 or OpenAttributeEntry1."""

    def __init__(self, attribute_names_dump_raw, version):
        self.buf = attribute_names_dump_raw

        if version >= 1:
            self.oat_class = OpenAttributeEntry1
        else:
            self.oat_class = OpenAttributeEntry0

    def find_file_reference_by_index(self, index):
        """Find and return the file reference using the entry index."""

        if index >= len(self.buf):
            raise ClientException('Invalid index: {}'.format(index))

        open_attribute_entry_buf = self.buf[index:]
        open_attribute_entry = self.oat_class(open_attribute_entry_buf)

        return open_attribute_entry.get_file_reference()

# dfir_ntfs: an NTFS parser for digital forensics & incident response # (c) <NAME> # # This module implements an interface to work with the $LogFile journal. # This is the size of a single memory page. # Values for the restart area: # No index in the array of client records. # Flags for the restart area: # Log entries are written one by one. # The new restart area has been written to the log file. # Flags for the log record page: # This page contains the end of a log record. # This page contains a client restart area (set when the log file version is 2.0 or higher). # Flags for the log record header: # This record was written to multiple pages. # This record contains no redo buffer. # This record contains no undo buffer. # Values for the log record types: # Operations for the NTFS client: # [1] # [1] # [1] # [1] # [1] # [1] # [2] # [1] # [1] # [1] # [1] # [2] # [2] # [2] # [1] # [2] # [1] # [2] # [2] # [2] # [1] # [2] # [1] # [2] # [1] # [1]: this is an update to the resident ($MFT) data. # [2]: this is an update to the nonresident data (which is referenced by an entry in the open attribute table). Convert an NTFS operation code to a string. Apply an update sequence array (USA) to multiple sectors (as a bytearray object) and return the (resulting buffer, end_of_usa_offset) tuple. # Check if integrity of data in this block is protected using the CRC32 value (for DAX volumes). # If the checksum value is invalid, treat this block as a usual one. # Skip the first element (sequence_number_in_usa_bytes). This is a top-level exception for this module. This exception is raised when something is wrong with an update sequence array. This exception is raised when something is wrong with a restart page. This exception is raised when the log file is empty. This exception is raised when something is wrong with a restart area. This exception is raised when something is wrong with a client record. This exception is raised when something is wrong with a log record page. This exception is raised when something is wrong with client data. This class is used to work with a restart page. Data of this restart page with updates from an update sequence array (USA) applied. Get and return the last log sequence number (LSN) found during the chkdsk scan. Get and return the system page size. This is the size of a restart page. Get and return the log page size. This is the size of a log record page. Get and return the restart area offset (relative from the start of this structure). Get, parse and return the restart area (as a RestartArea object). Get and return the major version of the log file. Get and return the minor version of the log file. This class is used to work with a restart area. Data of this restart area. Get and return the current log sequence number (LSN). Get and return the number of log clients. Get and return the index of the free list of client records. Get and return the index of the in-use list of client records. Get and return the flags (as an integer). Get and return the number of bits to use for the sequence number. Get and return the size of this restart area. Get and return the offset to the array of client records (relative from the start of this structure). Get and return the log file size. Get and return the size of data from the last log sequence number (LSN), excluding the header. Get and return the size of the log page header. Get and return the log page data offset. Get and return the revision number for this area. This method yields client records (ClientRecord objects). This class is used to work with a client record. Data of this client record. An allocated size of this record. # Do not cross the 'sizeof' boundary. Get and return the oldest log sequence number (LSN) required by this client. Get and return the log sequence number (LSN) of the latest client restart area. Get and return the index of the previous client record. Get and return the index of the next client record. Get and return the sequence number. Get and return the name length for this client. Get, decode and return the name of this client. This class is used to work with a log record page. Data of this log record page with updates from an update sequence array (USA) applied. Data of this log record page without updates from an update sequence array (USA) applied. Offset to the end of an update sequence array (USA) in the data. Offset to the start of the log data. Number of this page in the log file. Get and return the highest log sequence number (LSN) in this page. Get and return the log file offset for a tail copy. The log file version should be 1.1. This field is not valid for regular log record pages. # The same offset as above! Get and return the log file offset for a fast copy. The log file version should be 2.0 (or higher). This field is not valid for regular log record pages. Get and return the flags (as an integer). Get and return the page count. Get and return the page position. Get and return the offset of the free space in this page. Get and return the log sequence number (LSN) for the last record in this page. Get, parse and return a log record header (LogRecordHeader) by its offset in this log record page. #{}, USA end: {}, last LSN: {} (or file offset 1.1/2.x: {}/{}), flags: {}, page count: {}, page position: {}, last end LSN: {}'.format( This class is used to work with a log record header. Data of this log record (including client data). A size of this header (aligned to 8 bytes). Get and return the log sequence number (LSN) for this record. Get and return the previous log sequence number (LSN) for this client. Get and return the next undo log sequence number (LSN) for this client. Get and return the client data length for this record. Get and return the client ID for this record. Get and return the record type. Get and return the transaction ID. Get and return the flags (as an integer). This class is used to read and parse a $LogFile file. A file object for a $LogFile file. A size of this $LogFile file. The first restart page (a RestartPage object). The second restart page (a RestartPage object). A size of a single log record page. A tuple (major_version, minor_version) for this log file. A sorted list of log sequence numbers (LSNs) seen in this file. A dictionary with a sorted list of log sequence numbers (LSNs) grouped by client IDs (each client ID is a key in this dictionary). A number of bytes per cluster. (It will be adjusted later.) A version of the open attribute table to use. (It will be adjusted later.) # Both restart pages are corrupt. # The first restart page is corrupt, use the second one. # The second restart page is corrupt, use the first one. # Both restart pages are valid. # Adjusted in the 'collect_lsns' method. # Adjusted in the 'collect_lsns' method. Convert a log sequence number (LSN) to a file offset, return a (log_page_number, offset_in_page) tuple. Convert a log sequence number (LSN) to a sequence number and return it. Convert a log page number, an offset within that page, and a sequence number to a log sequence number (LSN) and return it. Get, parse and return a log record page (LogRecordPage) by its number (or None, if not found). # Truncated data. Return the first log record page number to be used after the log file is wrapped. # 2 restart pages and 2 tail log record pages. # 2 restart pages and 32 fast pages. Collect valid log sequence numbers (LSNs) found in this log file. # Validate every LSN found, collect new LSNs. # Validate these new LSNs. # Extend the list of LSNs found using new data. # Now, walk through all LSNs found before and locate new LSNs. Given a current LSN, find, validate and return the next LSN. If there is none or something is invalid, return None. # Client data is in the current log record page only. # Try the current log record page. # Try the next log record page. # Handle the wrap. # Client data is in two or more log record pages. # Client data length in the next pages (without the first one). # The number of pages to skip (including the first one with client data). # Handle the wrap. # Pages left after the first one with client data. # Try the next log record page. Give a current LSN, find, validate and return the previous LSN for the same client. If there is none or something is invalid, return None. Give a current LSN, find, validate and return the undo next LSN for the same client. If there is none or something is invalid, return None. # Find new LSNs by walking over client data. # Loop until a known LSN is encountered or something invalid is found. # Find new LSNs by looking for previous LSNs recorded for the same client. # Loop until a known LSN is encountered or something invalid is found. # Find new LSNs by looking for undo next LSNs for the same client. # Loop until a known LSN is encountered or something invalid is found. # Find new LSNs by looking in NTFS restart areas. # Adjust the cluster size. # Adjust the version-specific configuration. # Validate a new LSN. # Search for new LSNs. # Loop until a known LSN is encountered or something invalid is found. # Loop until a known LSN is encountered or something invalid is found. # Loop until a known LSN is encountered or something invalid is found. # Finally, store a sorted list of LSNs. Group log sequence numbers (LSNs) by their client IDs. Make a writable file object in memory (which is a copy of the original one) and use it for further operations. Discard all changes made to a writable file object (in memory) and use the original file object. Apply a tail log record page to the log file. The log file version should be 1.1. The file object should be writable. # Nothing to do, both tail log record pages are invalid. # Only the second tail log record page is present. # Only the first tail log record page is present. # Both tail log record pages are valid, choose the latest one. # A proper tail log record page was found. Apply fast log record pages to the log file. The log file version should be 2.0 or higher. The file object should be writable. # Locate the highest LSN present in the "infinite" area. # Sort the fast pages according to the log sequence numbers they have (fast pages with more recent log data should be written at the end). This method yields each log record page (LogRecordPage).         When 'mode' is 0, yield log record pages from the "infinite" (circular) area only.         When 'mode' is 1, yield each log record page from the whole file (not just from the "infinite" area). # 2 restart pages. Get, parse and return client data (ClientData) for a given log sequence number (LSN). # Client data is in the current log record page only. # Client data is in two or more log record pages. # Handle the wrap. # This page is missing in the current state of the log file. This method yields NTFS client data (NTFSRestartArea and NTFSLogRecord) from each log record found in the log file.         If 'recover_log_data' is True, apply a tail log record page or fast log record pages before parsing the log data. # Something is wrong, clear the open attribute table. This class is used to work with client data. Client data. A record type of this client data. A log sequence number (LSN) for this client data. A number of bytes per cluster. A transaction ID. Parse and return decoded data (NTFSRestartArea or NTFSLogRecord). Only the NTFS client is supported. This class is used work with an NTFS restart area. Data of this NTFS restart area. A log sequence number (LSN) of an NTFS restart area. Get and return the major version of a client. Get and return the minor version of a client. Get and return the log sequence number (LSN) of a checkpoint record. Get and return the log sequence number (LSN) of an open attribute table. Get and return the log sequence number (LSN) of attribute names. Get and return the log sequence number (LSN) of a dirty page table. Get and return the log sequence number (LSN) of a transaction table. Get and return the length of an open attribute table. Get and return the length of attribute names. Get and return the length of a dirty page table. Get and return the length of a transaction table. Get and return the absolute offset used as a starting point for checking update sequence number (USN) change journal records. Get and return the last value (as of the previous volume restart) of the "current" log sequence number (LSN). Get and return the number of bytes per cluster. Get and return the file record segment (FRS) reference for the update sequence number (USN) change journal. Get and return the base offset for the update sequence number (USN) change journal.         Since the $J data stream of this journal can be very large, this value is used to rebase offsets for high-level API functions (making them relative from the base offset). Get and return the oldest log sequence number (LSN) required by this client (as of the checkpoint). This class is used work with an NTFS log record. Data of this NTFS log record. A log sequence number (LSN) of this NTFS log record. An open attribute table (it is set externally in the 'parse_ntfs_records' method of the LogFileParser class). An open attribute table dump (as raw bytes; it is set externally in the 'parse_ntfs_records' method of the LogFileParser class). An attribute names dump (as raw bytes; it is set externally in the 'parse_ntfs_records' method of the LogFileParser class). A transaction ID. Get and return the redo operation code (as an integer). Get and return the undo operation code (as an integer). Get and return the offset of an undo record. Get and return the length of an undo record. Get and return the offset of an undo record. Get and return the length of an undo record. Get and return the open attribute table index for this update. Get and return the number of logical cluster numbers (LCNs) in use at the end of the header. Get and return the record offset. Get and return the attribute offset. Get and return the cluster block offset (this is a number of 512-byte blocks). For operations on the $MFT file, this field is set to the size (in bytes) of the file record segment (FRS) divided by 512.         If the attribute is $INDEX_ALLOCATION, this field is set to the size (in bytes) of the index record divided by 512.         In other cases, this field is set to 0. Get and return the target virtual cluster number (VCN). Get and return the array of logical cluster numbers (LCNs) as a list of integers. Get and return redo data. Get and return undo data. Calculate and return the number of a target file record segment (or None, if not applicable).         Note: this is for resident updates only. Calculate and return the offset within target data (or None, if not applicable). Calculate and return the reference to a target file record segment (FRS) and a target attribute name as a tuple (or None, if not applicable or not found).         Note: this is for nonresident updates present in an open attribute table. # If not found, try the table dumps. This class is used to work with the OPEN_ATTRIBUTE_ENTRY structure (version: 1.0+). Data of this OPEN_ATTRIBUTE_ENTRY structure. Get the free link (this value is 0xFFFFFFFF if this entry is allocated). Get and return the number of bytes per index buffer. Get and return the attribute type code. # At least one source states that there is a field between these two fields, but there is none. Get and return the file reference. Get and return the log sequence number (LSN) of an open log record (OpenNonresidentAttribute). # No field for the pointer is supported (because it is valid for a running machine only). This class is used to work with the OPEN_ATTRIBUTE_ENTRY structure (version: 0.0). Data of this OPEN_ATTRIBUTE_ENTRY structure. Get the free link (this value is 0xFFFFFFFF if this entry is allocated). Get and return the table index (offset) of an original (version: 1.0+) entry (in a table in memory). Get and return the file reference. Get and return the log sequence number (LSN) of an open log record (OpenNonresidentAttribute). Get and return the attribute type code. Get and return the number of bytes per index buffer. This class is used to work with the ATTRIBUTE_NAME_ENTRY structure. Data of this ATTRIBUTE_NAME_ENTRY structure. Get and return the table index (offset) of a corresponding entry in the open attribute table. Get and return the name length. Get and return the name. Calculate and return the size of this entry. # Include the terminating null character. This class is used to work with the array of ATTRIBUTE_NAME_ENTRY structures. Data of this array. Find and return the name by its index (or None, if not found). This class is used to work with the array of OPEN_ATTRIBUTE_ENTRY structures. Data of this array. Either OpenAttributeEntry0 or OpenAttributeEntry1. Find and return the file reference using the entry index.