"""
Created on 30 Jul 2017

@author: jdrumgoole
"""
import socket
import sys
from datetime import datetime
from enum import Enum

import pymongo

from pymongoimport.canonical_path import Canonical_Path


class Restart_State(Enum):
    undefined = 0
    start = 1
    inprogress = 2
    finish = 3


class Restarter(object):
    """
    Track insertion of a collection of docs by adding the last inserted
    ID into a collection called "restartlog". Each time we insert we add
    a doc with a timestamp and an ID field and a count of the number of
    entries inserted to date. The doc also contains a batch
    start time.

    These class assumes the object ID is defined by default as per the MongoDB docs
    (https://docs.mongodb.com/manual/reference/method/ObjectId/). In this case in a single run
    of the pymongoimport the IDs will contain identical host and process components. We can use
    these fields to identify inserts that happened in the previous run. So we search for all inserts
    with an ID greater than the ID in the restartlog.

    We then scan that list of inserts for a matching ID
    (remember we may be running multiple batch uploads in parallel) to find the inserts related to this
    batch restart. Once we have that list of matching inserts then we have the count of objects inserted.
    Now we know where the restart position should be (restartlog number of entries + len( list of matching inserts)
    We can now skip() to that position in the input file and update the restart log.

    """

    def __init__(self, database, input_filename, batch_size, cmd=None):
        """
        Constructor
        """
        self._audit = database["AUDIT"]
        self._name = Canonical_Path(input_filename)
        self._batch_size = batch_size
        self._hostname = socket.gethostname()
        if cmd is None:
            self._cmd = " ".join(sys.argv)
        else:
            self._cmd = cmd

        self._restartDoc = self._audit.find_one({"name": self._name(),
                                                 "state": "inprogress"})

        if self._restartDoc is None:
            self.start()

    @staticmethod
    def split_ID(doc_id):
        """
        Split a MongoDB Object ID
        a 4-byte value representing the seconds since the Unix epoch,
        a 3-byte machine identifier,
        a 2-byte process id, and
        A 3-byte counter, starting with a random value.
        """
        id_str = str(doc_id)
        #        epoch 0        machine   1    process ID  2    counter 3
        return (id_str[0:8], id_str[8:14], id_str[14:18], id_str[18:24])

    def start(self):
        self._audit.insert_one({"name": self._name(),
                                "ts": datetime.utcnow(),
                                "batch_size": self._batch_size,
                                "command": self._cmd,
                                "state": Restart_State.start})

    def update(self, doc_id, count):

        self._audit.insert_one({"name": self._name(),
                                "count": count,
                                "ts": datetime.utcnow(),
                                "doc_id": doc_id,
                                "state": Restart_State.inprogress})

    def restart(self, collection):
        """
        Get the restart doc. Now find any docs created after the restart doc was created
        within the same process and machine. Count those so we know where we are.
        Return the new doc count that we can skip too.
        """

        self._restartDoc = self._audit.find_one({"name": self._name(),
                                                 "state": Restart_State.inprogress})

        if self._restartDoc is None:  # skip nothing, nothing to restart
            return 0

        count = self._restartDoc["count"]
        (_, machine, pid, _) = Restarter.split_ID(self._restartDoc["doc_id"])

        cursor = collection.find({"_id": {"$gt": self._restartDoc["doc_id"]}})

        for i in cursor:
            (_, i_machine, i_pid, _) = Restarter.split_ID(i["_id"])

            if i_machine == machine and i_pid == pid:
                count = count + 1

            if count == self._restartDoc["batch_size"]:
                # we have the full batch, we can't have inserted more than 
                # this before updating the restart doc
                break

        return count

    def finish(self):

        self._restartDoc = self._audit.insert_one({"name": self._name(),
                                                   "ts": datetime.utcnow(),
                                                   "state": Restart_State.finish})

    def _find_last(self, col, doc):

        if "ts" in doc:
            cursor = col.find(doc).sort({"ts": pymongo.DESCENDING}).limit(1)
            for c in cursor:
                return c
            return None
        else:
            raise ValueError("_find_last requires a timestamp field 'ts'")

    def get_state(self, name):

        doc = self._audit.find({"name": name,
                                "state": Restart_State.finish}).sort({"ts": pymongo.DESCENDING}).limit(1)

        if doc:
            return Restart_State.finish

        doc = self._audit.find_one({"name": name,
                                    "state": Restart_State.inprogress})

        if doc:
            return Restart_State.inprogress

        doc = self._audit.find_one({"name": name,
                                    "state": Restart_State.start})

    def reset(self):

        self._restartDoc = self._audit.find_one_and_update({"name": self._name()},
                                                           {"$set": {"timestamp": datetime.utcnow(),
                                                                     "batch_size": self._batch_size,
                                                                     "count": 0,
                                                                     "last_doc_id": 0,
                                                                     "state": "inprogress"}})

"""
Created on 30 Jul 2017

@author: jdrumgoole
"""
import socket
import sys
from datetime import datetime
from enum import Enum

import pymongo

from pymongoimport.canonical_path import Canonical_Path


class Restart_State(Enum):
    undefined = 0
    start = 1
    inprogress = 2
    finish = 3


class Restarter(object):
    """
    Track insertion of a collection of docs by adding the last inserted
    ID into a collection called "restartlog". Each time we insert we add
    a doc with a timestamp and an ID field and a count of the number of
    entries inserted to date. The doc also contains a batch
    start time.

    These class assumes the object ID is defined by default as per the MongoDB docs
    (https://docs.mongodb.com/manual/reference/method/ObjectId/). In this case in a single run
    of the pymongoimport the IDs will contain identical host and process components. We can use
    these fields to identify inserts that happened in the previous run. So we search for all inserts
    with an ID greater than the ID in the restartlog.

    We then scan that list of inserts for a matching ID
    (remember we may be running multiple batch uploads in parallel) to find the inserts related to this
    batch restart. Once we have that list of matching inserts then we have the count of objects inserted.
    Now we know where the restart position should be (restartlog number of entries + len( list of matching inserts)
    We can now skip() to that position in the input file and update the restart log.

    """

    def __init__(self, database, input_filename, batch_size, cmd=None):
        """
        Constructor
        """
        self._audit = database["AUDIT"]
        self._name = Canonical_Path(input_filename)
        self._batch_size = batch_size
        self._hostname = socket.gethostname()
        if cmd is None:
            self._cmd = " ".join(sys.argv)
        else:
            self._cmd = cmd

        self._restartDoc = self._audit.find_one({"name": self._name(),
                                                 "state": "inprogress"})

        if self._restartDoc is None:
            self.start()

    @staticmethod
    def split_ID(doc_id):
        """
        Split a MongoDB Object ID
        a 4-byte value representing the seconds since the Unix epoch,
        a 3-byte machine identifier,
        a 2-byte process id, and
        A 3-byte counter, starting with a random value.
        """
        id_str = str(doc_id)
        #        epoch 0        machine   1    process ID  2    counter 3
        return (id_str[0:8], id_str[8:14], id_str[14:18], id_str[18:24])

    def start(self):
        self._audit.insert_one({"name": self._name(),
                                "ts": datetime.utcnow(),
                                "batch_size": self._batch_size,
                                "command": self._cmd,
                                "state": Restart_State.start})

    def update(self, doc_id, count):

        self._audit.insert_one({"name": self._name(),
                                "count": count,
                                "ts": datetime.utcnow(),
                                "doc_id": doc_id,
                                "state": Restart_State.inprogress})

    def restart(self, collection):
        """
        Get the restart doc. Now find any docs created after the restart doc was created
        within the same process and machine. Count those so we know where we are.
        Return the new doc count that we can skip too.
        """

        self._restartDoc = self._audit.find_one({"name": self._name(),
                                                 "state": Restart_State.inprogress})

        if self._restartDoc is None:  # skip nothing, nothing to restart
            return 0

        count = self._restartDoc["count"]
        (_, machine, pid, _) = Restarter.split_ID(self._restartDoc["doc_id"])

        cursor = collection.find({"_id": {"$gt": self._restartDoc["doc_id"]}})

        for i in cursor:
            (_, i_machine, i_pid, _) = Restarter.split_ID(i["_id"])

            if i_machine == machine and i_pid == pid:
                count = count + 1

            if count == self._restartDoc["batch_size"]:
                # we have the full batch, we can't have inserted more than 
                # this before updating the restart doc
                break

        return count

    def finish(self):

        self._restartDoc = self._audit.insert_one({"name": self._name(),
                                                   "ts": datetime.utcnow(),
                                                   "state": Restart_State.finish})

    def _find_last(self, col, doc):

        if "ts" in doc:
            cursor = col.find(doc).sort({"ts": pymongo.DESCENDING}).limit(1)
            for c in cursor:
                return c
            return None
        else:
            raise ValueError("_find_last requires a timestamp field 'ts'")

    def get_state(self, name):

        doc = self._audit.find({"name": name,
                                "state": Restart_State.finish}).sort({"ts": pymongo.DESCENDING}).limit(1)

        if doc:
            return Restart_State.finish

        doc = self._audit.find_one({"name": name,
                                    "state": Restart_State.inprogress})

        if doc:
            return Restart_State.inprogress

        doc = self._audit.find_one({"name": name,
                                    "state": Restart_State.start})

    def reset(self):

        self._restartDoc = self._audit.find_one_and_update({"name": self._name()},
                                                           {"$set": {"timestamp": datetime.utcnow(),
                                                                     "batch_size": self._batch_size,
                                                                     "count": 0,
                                                                     "last_doc_id": 0,
                                                                     "state": "inprogress"}})

Created on 30 Jul 2017  @author: jdrumgoole Track insertion of a collection of docs by adding the last inserted     ID into a collection called "restartlog". Each time we insert we add     a doc with a timestamp and an ID field and a count of the number of     entries inserted to date. The doc also contains a batch     start time.      These class assumes the object ID is defined by default as per the MongoDB docs     (https://docs.mongodb.com/manual/reference/method/ObjectId/). In this case in a single run     of the pymongoimport the IDs will contain identical host and process components. We can use     these fields to identify inserts that happened in the previous run. So we search for all inserts     with an ID greater than the ID in the restartlog.      We then scan that list of inserts for a matching ID     (remember we may be running multiple batch uploads in parallel) to find the inserts related to this     batch restart. Once we have that list of matching inserts then we have the count of objects inserted.     Now we know where the restart position should be (restartlog number of entries + len( list of matching inserts)     We can now skip() to that position in the input file and update the restart log. Constructor Split a MongoDB Object ID         a 4-byte value representing the seconds since the Unix epoch,         a 3-byte machine identifier,         a 2-byte process id, and         A 3-byte counter, starting with a random value. #        epoch 0        machine   1    process ID  2    counter 3 Get the restart doc. Now find any docs created after the restart doc was created         within the same process and machine. Count those so we know where we are.         Return the new doc count that we can skip too. # skip nothing, nothing to restart # we have the full batch, we can't have inserted more than # this before updating the restart doc