class MPEngineProdCons(object):

    def __init__(self, maxCores, producer_Class, consumer_Class, governorOffFlag = False):
        logger.debug("mpEngine initializing")
        self.governorOffFlag = governorOffFlag
        self.maxCores = maxCores
        self.__deleting__ = False
        self.__internalLock__ = multiprocessing.Lock()
        self.killed_event = multiprocessing.Event()

        # Producers
        self.num_producers = 0
        self.next_worker_num = 0
        self.producer_Class = producer_Class
        self.producer_pool = []
        self.producer_pool_exitEvent = []
        self.producer_task_queue = multiprocessing.JoinableQueue()
        self.producer_results_queue = multiprocessing.JoinableQueue()
        self.producer_pool_progress = multiprocessing.Value('i', 0)

        # Consumers
        self.num_consumers = 0
        self.next_consumer_num = 0
        self.consumer_Class = consumer_Class
        self.consumer_pool = []
        # Note: consumer_pool_exitEvent is used both to notify a worker it should end and for the worker to notify it has dones so
        self.consumer_pool_exitEvent = []
        self.consumer_task_queue = self.producer_results_queue
        self.consumer_results_queue = multiprocessing.JoinableQueue()
        self.consumer_pool_progress = multiprocessing.Value('i', 0)

        # Tasks
        self.num_tasks = multiprocessing.Value('i', 0)
        self.tasks_added = False

        # Rebalance checks
        self._rebalance_last_kick = datetime.now()
        self.rebalance_backoff_timer = 60 * 1
        self._rebalance_mem_last_kick = datetime.now()
        self.rebalance_mem_backoff_timer = 60 * 2

    def __enter__(self):
        return self


    def __exit__(self, exc_type, exc_value, traceback):
        if exc_type is not None:
            print exc_type, exc_value, traceback

        self.__del__
        return self


    def __del__(self):
        # Lock internal
        self.__internalLock__.acquire()
        if not self.__deleting__:
            logger.debug("Bringing down mpEngine")
            self.__deleting__ = True

            while self.num_producers > 0: self.removeProducer(True)
            while self.num_consumers > 0: self.removeConsumer(True)

            logger.debug("mpEngine down")
        # Release internal
        self.__internalLock__.release()


    def check_mpEngineStatus(self):
        status_ok = True
        assert(len(self.producer_pool) == self.num_producers)
        assert(len(self.producer_pool) == len(self.producer_pool_exitEvent))
        assert(len(self.consumer_pool) == self.num_consumers)
        assert(len(self.consumer_pool) == len(self.consumer_pool_exitEvent))
        # Check all the processes we believe we have are really alive
        for (worker_num, worker, extra_arg_list) in self.producer_pool:
            if not worker.is_alive():
                logger.error("check_mpEngineStatus error, dead producer process: %s / %s" % (worker_num, worker.name))
                status_ok = False

        for (worker_num, worker, extra_arg_list) in self.consumer_pool:
            if not worker.is_alive():
                logger.error("check_mpEngineStatus error, dead consumer process: %s / %s" % (worker_num, worker.name))
                status_ok = False

        if self.killed_event.is_set(): status_ok = False
        return status_ok


    def working(self):
        # Internal check
        if self.killed_event.is_set() or not self.check_mpEngineStatus():
            print("\n\n\n==>Tearing down mpEngine, we've been killed! out of memory? (give me 60 secs to try to shutdown cleanly)")
            logger.error("Tearing down mpEngine, we've been killed!")
            self.endConsumers()
            time.sleep(2)
            self.endProducers()
            time.sleep(2)
            return False

        # Check if we still have work to do
        if (self.get_num_tasks() != self.getProgressConsumers()): return True
        else:
            logger.debug("mpEngine work finished!")
            # Wait until all our workers have exited
            while self.num_producers > 0: self.removeProducer(True)
            while self.num_consumers > 0: self.removeConsumer(True)
            return False


    def addTaskList(self, task_list):
        if not self.tasks_added:
            with self.num_tasks.get_lock():
                for new_task in task_list:
                    self.num_tasks.value += 1
                    self.producer_task_queue.put(new_task)
            return True
        else:
            logger.error("Can only add tasks once!")
            return False


    def getProgress(self):
        return (self.num_producers, self.num_consumers, self.get_num_tasks(), self.getProgressProducers(), self.getProgressConsumers())


    def get_num_tasks(self):
        with self.num_tasks.get_lock():
            return self.num_tasks.value


    def getProgressProducers(self):
        return self.producer_pool_progress.value


    def getProgressConsumers(self):
        return self.consumer_pool_progress.value


    def getProducerCount(self):
        return self.num_producers


    def addProducer(self, extra_arg_list = []):
        if self.num_producers < self.maxCores:
            # Lock internal
            self.__internalLock__.acquire()

            new_worker_num = self.next_worker_num
            logger.debug("Adding Producer-%d" % (new_worker_num))
            self.producer_pool_exitEvent.append(multiprocessing.Event())
            self.producer_pool.append((new_worker_num, self.producer_Class(
                self.producer_task_queue, self.producer_results_queue, self.get_num_tasks(), self.get_num_tasks(), self.producer_pool_progress,
                self.producer_pool_exitEvent[-1], self.killed_event, extra_arg_list), extra_arg_list))
            self.producer_pool[-1][1].daemon = False # Remove for debugging
            self.producer_pool[-1][1].start()

            # Update worker count
            self.num_producers += 1

            # Update next worker num
            self.next_worker_num += 1

            # Release internal
            self.__internalLock__.release()

            logger.debug("Producer-%d added" % new_worker_num)
        else:
            logger.error("Attempted to start workers beyond the maxCores setting")


    def removeProducer(self, noLock = False):
        if self.num_producers > 0:
            # Lock internal
            if not noLock: self.__internalLock__.acquire()

            # Remove last worker from worker pool
            (worker_num, producer, extra_arg_list) = self.producer_pool.pop()
            logger.debug("Removing Producer-%d" % worker_num)
            # Remove last worker's exitFlag
            producer_exitEvent = self.producer_pool_exitEvent.pop()

            # Set the worker's exit event
            if not producer_exitEvent.is_set():
                logger.debug("Producer-%d exitEvent SET" % worker_num)
                producer_exitEvent.set()

            # Update producer count
            self.num_producers -= 1

            # Release internal
            if not noLock: self.__internalLock__.release()
        else:
            logger.error("Attempted to remove producer from empty pool.")


    def startProducers(self, num_producers, extra_arg_list = []):
        logger.debug("Starting producers")
        if num_producers is None:
            for i in xrange(self.maxCores - 1): self.addProducer(extra_arg_list)
        else:
            for i in xrange(num_producers): self.addProducer(extra_arg_list)


    def restartProducers(self):
        logger.debug("Restarting producers")
        extra_arg_list_list = []
        current_num_producers = self.num_producers
        # Shut them all down
        for i in xrange(current_num_producers):
            # Grab extra_arg_list
            (worker_num, producer, extra_arg_list) = self.producer_pool[-1]
            extra_arg_list_list.append(extra_arg_list)
            self.removeProducer(True)
        # Start them all up again
        for i in xrange(current_num_producers):
            self.addProducer(extra_arg_list_list.pop())

        logger.debug("Restarting producers - done")


    def endProducers(self):
        logger.debug("Ending all producers")
        for i in xrange(self.num_producers): self.removeProducer()


    def getConsumerCount(self):
        return self.num_consumers


    def addConsumer(self, extra_arg_list = []):
        if self.num_consumers < self.maxCores:
            # Lock internal
            self.__internalLock__.acquire()

            new_worker_num = self.next_worker_num
            logger.debug("Adding Consumer-%d" % (new_worker_num))
            self.consumer_pool_exitEvent.append(multiprocessing.Event())
            self.consumer_pool.append((new_worker_num, self.consumer_Class(
                self.consumer_task_queue, self.consumer_results_queue, self.get_num_tasks(), self.producer_pool_progress, self.consumer_pool_progress,
                self.consumer_pool_exitEvent[-1], self.killed_event, extra_arg_list), extra_arg_list))
            self.consumer_pool[-1][1].daemon = False  # Remove for debugging
            self.consumer_pool[-1][1].start()

            # Update consumer count
            self.num_consumers += 1

            # Update next worker num
            self.next_worker_num += 1

            # Release internal
            self.__internalLock__.release()

            logger.debug("Consumer-%d added" % new_worker_num)
        else:
            logger.error("Attempted to start workers beyond the maxCores setting")


    def removeConsumer(self, noLock = True):
        if self.num_consumers > 0:
            # Lock internal
            if not noLock: self.__internalLock__.acquire()

            # Remove last worker from worker pool
            (worker_num, consumer, extra_arg_list) = self.consumer_pool.pop()
            logger.debug("Removing Consumer-%d" % worker_num)
            # Remove last worker's exitFlag
            consumer_exitEvent = self.consumer_pool_exitEvent.pop()

            # Set the worker's exit event
            if not consumer_exitEvent.is_set():
                logger.debug("Consumer-%d exitEvent SET" % worker_num)
                consumer_exitEvent.set()
                # Wait for the worker to acknowledge he has shutdown:
                while consumer_exitEvent.is_set():
                    logger.debug("Waiting for Consumer-%d to shutdown" % worker_num)
                    time.sleep(1)

            # Update consumer count
            self.num_consumers -= 1

            # Release internal
            if not noLock: self.__internalLock__.release()
        else:
            logger.error("Attempted to remove consumer from empty pool.")


    def startConsumers(self, num_consumers, extra_arg_list = []):
        logger.debug("Starting consumers")
        if num_consumers is None:
            for i in xrange(self.maxCores - 1): self.addConsumer(extra_arg_list)
        else:
            for i in xrange(num_consumers): self.addConsumer(extra_arg_list)


    def restartConsumers(self):
        logger.debug("Restarting consumers")
        extra_arg_list_list = []
        current_num_consumers = self.num_consumers
        # Shut them all down
        for i in xrange(current_num_consumers):
            # Grab extra_arg_list
            (worker_num, consumer, extra_arg_list) = self.consumer_pool[-1]
            extra_arg_list_list.append(extra_arg_list)
            self.removeConsumer(True)

        # Give them time to actually shutdown
        time.sleep(1)
        # Start them all up again
        for i in xrange(current_num_consumers):
            self.addConsumer(extra_arg_list_list.pop())

        logger.debug("Restarting consumers - done")


    def endConsumers(self):
        logger.debug("Ending all consumers")
        for i in xrange(self.num_consumers): self.removeConsumer()


    def grabResults(self):
        results = []
        try:
            while True:
                next_result = self.consumer_results_queue.get_nowait()
                results.append(next_result)
        except Queue.Empty:
            pass

        return results


    @rate_limited(1.0 / 10.0)
    def rebalance(self):
        if self.governorOffFlag:
            return
        progProducers = self.getProgressProducers()
        progConsumers = self.getProgressConsumers()
        num_tasks = self.get_num_tasks()
        elapsed_backoff_time = (datetime.now() - self._rebalance_last_kick).seconds

        logger.debug("Starting balancing (timer: %d/%d)" % (elapsed_backoff_time, self.rebalance_backoff_timer))

        # Kill producers if all tasks have been served
        if num_tasks == progProducers and progProducers > 0:
            self.endProducers()
            return

        # Restart paused production on backoff timer or if we have at least 20% memory available
        if self.num_producers == 0 and ((elapsed_backoff_time > self.rebalance_backoff_timer) or psutil_phymem_usage() < 80):
            logger.debug("Rebalancing, restarting production")
            self.addProducer()
            return

        # Memory governor
        # Pause production if we're over 90%
        if psutil_phymem_usage() > 90:
            logger.debug("Rebalancing, mem > 90%, pausing production")
            self.endProducers()
            self._rebalance_last_kick = datetime.now()
            return
        # Reduce production if we're over 75%
        if psutil_phymem_usage() > 75 and self.num_producers > 1:
            if (datetime.now() - self._rebalance_mem_last_kick).seconds > self.rebalance_mem_backoff_timer:
                logger.debug("Rebalancing, mem > 75%")
                self.removeProducer()
                self._rebalance_mem_last_kick = datetime.now()
                return

        # Memory pressure check
        if psutil_phymem_usage() > 70:
            if (datetime.now() - self._rebalance_mem_last_kick).seconds > self.rebalance_mem_backoff_timer:
                logger.debug("Rebalancing mem, recycling processes")
                self.restartConsumers()
                self.restartProducers()
                gc.collect()
                self._rebalance_mem_last_kick = datetime.now()
                logger.debug("Rebalancing mem, recycling processes - done")
                return
            else:
                logger.debug("Rebalance (Memory pressure check) postponed, waiting for rebalance_backoff_timer")

        # We wait until tasks are moving along to start rebalancing stuff
        if progProducers < (num_tasks / 10):
            return

        # Auto-balancing
        if progProducers > progConsumers * 2:
            if self.num_producers > 1:
                if elapsed_backoff_time > self.rebalance_backoff_timer:
                    logger.debug("Rebalancing, too many producers")
                    self.removeProducer()
                    self._rebalance_last_kick = datetime.now()
                    return
                else: logger.debug("Rebalance postponed, waiting for rebalance_backoff_timer")
        elif progProducers < progConsumers * 1.20:
            if num_tasks > progProducers * 1.20:
                if psutil_phymem_usage() < 70 and elapsed_backoff_time > self.rebalance_backoff_timer:
                    logger.debug("Rebalancing")
                    self.addProducer()
                    self._rebalance_last_kick = datetime.now()
                    return
                else: logger.debug("Rebalance (Auto-balancing) postponed, waiting for rebalance_backoff_timer")

        logger.debug("Balancing done") class MPEngineWorker(multiprocessing.Process):

    def __init__(self, task_queue, results_queue, total_task_num, available_task_num, progress_counter, exitEvent, killed_event, extra_arg_list):
        multiprocessing.Process.__init__(self)
        # logging.config.dictConfig(logger_config_worker)
        # if os.name == 'posix':
        #     # On POSIX, the setup logger will have been configured in the
        #     # parent process, but should have been disabled following the
        #     # dictConfig call.
        #     # On Windows, since fork isn't used, the setup logger won't
        #     # exist in the child, so it would be created and the message
        #     # would appear - hence the "if posix" clause.
        #     self.logger = logging.getLogger('setup')
        #     self.logger.critical('worker_process - Should not appear, because of disabled logger ...')

        self.proc_name = self.name.split("-")[0]+"-"+str(int(self.name.split("-")[1]) - 1)
        self.logger = logging.getLogger(self.proc_name)
        self.logger.debug("%s - __init__" % self.proc_name)

        self.task_queue = task_queue
        self.results_queue = results_queue
        self.total_task_num = total_task_num
        self.available_task_num = available_task_num
        self.progress_counter = progress_counter
        self.exitEvent = exitEvent
        self.killed_event = killed_event

        assert(type(extra_arg_list) is list)
        self.extra_arg_list = extra_arg_list

        self.killer = GracefulKiller()

    def __del__(self):
        self.logger.debug("%s - __del__" % self.proc_name)

    def update_progress(self):
        self.logger.debug("%s - update_progress" % self.proc_name)
        self.task_queue.task_done()
        # Update progress
        with self.progress_counter.get_lock():
            self.progress_counter.value += 1

        # Check if we've been killed
        self.check_killed()

    def check_killed(self):
        # Check if they've tried to kill us
        if self.killer.kill_now:
            self.logger.error("+++%s - KILLED" % self.proc_name)
            self.killed_event.set()

    def run(self):
        self.logger.debug("%s - starting work! PID: %d" % (self.proc_name, self.pid))
        while not self.exitEvent.is_set():
            try:
                self.logger.debug("%s - trying to grab task" % self.proc_name)
                next_task = self.task_queue.get_nowait()
            except Queue.Empty:
                if self.exitEvent.is_set(): break
                else:
                    self.logger.debug("%s - no tasks but we're not ready to die, sleeping" % self.proc_name)
                    time.sleep(1)
                    continue

            self.logger.debug("%s - working" % self.proc_name)
            result = self.do_work(next_task)
            # Update progress
            self.update_progress()
            # Write results to the results_queue
            self.results_queue.put(result)

        # We clear the our exitEvent to indicate we've received it and will shutdown
        self.exitEvent.clear()
        self.logger.debug("%s - finished!" % self.proc_name)
        time.sleep(1) def update_progress(progress, text="Progress", logmessage=False):
    if not stdout_redirect():
        oh = sys.stdout
    else:
        oh = sys.stderr

    barLength = 25  # Modify this to change the length of the progress bar
    status = ""
    if isinstance(progress, int):
        progress = float(progress)
    if not isinstance(progress, float):
        progress = 0
        status = "error: progress var must be float\r\n"
    if progress < 0:
        progress = 0
        status = "Halt...\r\n"
    if progress > 1:
        status = "Progress out of bounds...\r\n"
    if progress == 1:
        progress = 1
        oh.write('\x1b[2K\r')
        oh.flush()
        return ""
    block = int(round(barLength * progress))
    if logmessage:
        text = "{0}: [{1}] {2}% {3}\033[K".format(text, "#" * block + "-" * (barLength - block), round(progress * 100, 2), status)
        return text
    else:
        text = "\r{0}: [{1}] {2}% {3}\033[K".format(text, "#" * block + "-" * (barLength - block), round(progress * 100, 2), status)
        oh.write(text)
        oh.flush() def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in xrange(0, len(l), n):
        yield l[i:i + n] def loadFile(file_fullpath, max_chunk_size = 0):
    """Abstracts loading a regular file and a file from within a zip archive.
    Args:
        file_fullpath (str): Full path to file to load
        max_chunk_size (int): maximum number of bytes to be read from the file
    Returns:
        file_pointer (StringIO): Data read from fileFullPath
    """
    global zipCache
    global zipCache_hits
    global zipCache_misses
    logger.debug("Loading file %s" % file_fullpath)
    if ".zip/" in file_fullpath:
        m = re.match(r'^((?:.*)\.zip)[\\/](.*)$', file_fullpath)
        if m:
            zip_container = m.group(1)
            if m.lastindex > 1:
                file_fullpath_unprocessed = m.group(2)
                if '.zip' in file_fullpath_unprocessed:
                    m = re.match(r'^(.*)(?:\.zip[\\/](.*))*$', file_fullpath_unprocessed)
                    if m:
                        zip_container_relative_path = m.group(1)
                        if m.lastindex > 1:
                            file_fullpath_unprocessed = m.group(2)
                        else:
                            file_fullpath_unprocessed = None
                    else:
                        logger.error("Issue extracting container and relative path from ZIP file: %s" % file_fullpath)
                else:
                    zip_container_relative_path = file_fullpath_unprocessed

                # If not in the zipCache we add it
            if zip_container not in zipCache:
                zipCache_misses +=1
                if zipfile.is_zipfile(loadFile(zip_container)):
                    zipCache[zip_container] = zipfile.ZipFile(loadFile(zip_container))
                else:
                    logger.error("Invalid ZIP file found: %s" % file_fullpath)
                    return None
            else:
                zipCache_hits += 1
                # print("ZipCache hits / misses: %d/%d" % (zipCache_hits, zipCache_misses))

            # Extract file_pointer from zip:
            file_pointer = StringIO(zipCache[zip_container].read(zip_container_relative_path))
            if max_chunk_size != 0:
                file_pointer = StringIO(file_pointer.read(max_chunk_size))

        else:
            logger.error("Issue extracting container and relative path from ZIP file: %s" % file_fullpath)
    else:
            # Extract file_pointer:
            input_file = open(file_fullpath, 'rb')
            if max_chunk_size == 0:
                file_pointer = StringIO(input_file.read())
            else: file_pointer = StringIO(input_file.read(max_chunk_size))
    assert(file_pointer is not None)

    logger.debug("Read %d bytes [%s]" % (getFileSize(file_pointer), toHex(file_pointer.read(20))))
    # Return file pointer to pos 0
    file_pointer.seek(0, 0)

    return file_pointer def psutil_phymem_usage():
    """
    Return physical memory usage (float)
    Requires the cross-platform psutil (>=v0.3) library
    (http://code.google.com/p/psutil/)
    """
    # This is needed to avoid a deprecation warning error with
    # newer psutil versions

    if settings.__PSUTIL__ == False:
        return 0.0

    try:
        percent = psutil.virtual_memory().percent
    except:
        percent = psutil.phymem_usage().percent
    return percent def file_size(file_name_fullpath):
    if ".zip/" in file_name_fullpath:
        file_object = loadFile(file_name_fullpath)
        file_object.seek(0, os.SEEK_END)
        return file_object.tell()
    else: return os.path.getsize(file_name_fullpath) def read_mir(xml_file, quiet=False):
    out_list = []
    tmp_list = []
    error = ""

    # Open the MIR output file.
    try:
        for (_, reg_item) in et.iterparse(xml_file, events=('end',)):
            if reg_item.tag != 'RegistryItem':
                continue

            path_name = reg_item.find("Path").text
            if not path_name:
                print "[-] Error XML missing Path"
                print et.tostring(reg_item)
                reg_item.clear()
                continue
            path_name = path_name.lower()

            # Check to see that we have the right registry value.
            if 'control\\session manager\\appcompatcache\\appcompatcache' in path_name \
                or 'control\\session manager\\appcompatibility\\appcompatcache' in path_name:
                # return the base64 decoded value data.
                bin_data = binascii.a2b_base64(reg_item.find('Value').text)
                tmp_list = read_cache(bin_data, quiet)

                if tmp_list:
                    for row in tmp_list:
                        if g_verbose:
                            row.append(path_name)
                        if row not in out_list:
                            out_list.append(row)
            reg_item.clear()
    except (AttributeError, TypeError, IOError),  err:
        error = "[-] Error reading MIR XML: %s" % str(err)
        print error
        return (error, None)

    if len(out_list) == 0:
        return (error, None)
    else:
        # Add the header and return the list.
        if g_verbose:
            out_list.insert(0, output_header + ['Key Path'])
        else:
        # Only return unique entries.
            out_list = unique_list(out_list)
            out_list.insert(0, output_header)

    return (error, out_list) def write_it(rows, outfile=None, quiet=True):

    try:

        if not rows:
            print "[-] No data to write..."
            return

        if not outfile:
            for row in rows:
                print " ".join(["%s"%x for x in row])
        else:
            if not quiet: print "[+] Writing output to %s..."%outfile
            try:
                if "StringIO" in outfile:
                    return_rows = []
                    for row in rows:
                        return_rows.append(",".join(["%s" % x for x in row]))
                    return return_rows
                else:
                    f = open(outfile, 'wb')
                    if g_usebom:
                        f.write(codecs.BOM_UTF8)
                    csv_writer = writer(f, delimiter=',')
                    csv_writer.writerows(rows)
                    f.close()
            except IOError, err:
                print "[-] Error writing output file: %s" % str(err)
                return

    except UnicodeEncodeError, err:
        print "[-] Error writing output file: %s" % str(err)
        return def _processAmCacheFile_StringIO(data):
    regf_file = pyregf.file()
    regf_file.open_file_object(data, "r")

    try:
        ee = parse_execution_entries(regf_file)
    except NotAnAmcacheHive:
        logger.error("doesn't appear to be an Amcache.hve hive")
        return
    finally:
        regf_file.close()

    return ee class Issues_document(Ingest):
    def __init__(self):
    def checkMagic(self, file_name_fullpath):
    def calculateID(self, file_name_fullpath):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Appcompat_hxregistryaudit(Ingest):
    def __init__(self):
    def calculateID(self, file_name_fullpath):
    def checkMagic(self, file_name_fullpath):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Appcompat_parsed(Ingest):
    def __init__(self):
    def checkMagic(self, file_name_fullpath):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Appcompat_mirregistryaudit(Ingest):
    def __init__(self):
    def calculateID(self, file_name_fullpath):
    def checkMagic(self, file_name_fullpath):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Amcache_miracquisition(Ingest):
    def __init__(self):
    def getHostName(self, file_name_fullpath):
    def checkMagic(self, file_name_fullpath):
    def calculateID(self, file_name_fullpath):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Appcompat_mirlua_v1(Ingest):
    def __init__(self):
    def calculateID(self, file_name_fullpath):
    def checkMagic(self, file_name_fullpath):
    def _processElement(self, element, tag_dict, tag_prefix = ""):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Appcompat_mirlua_v2(Ingest):
    def __init__(self):
    def checkMagic(self, file_name_fullpath):
    def calculateID(self, file_name_fullpath):
    def _processElement(self, element, tag_dict, tag_prefix = ""):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Amcache_mirlua_v1(Ingest):
    def __init__(self):
    def calculateID(self, file_name_fullpath):
    def checkMagic(self, file_name_fullpath):
    def _processElement(self, element, tag_dict, tag_prefix = ""):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Appcompat_csv(Ingest):
    def __init__(self):
    def checkMagic(self, file_name_fullpath):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Appcompat_redline(Ingest):
    def __init__(self):
    def calculateID(self, file_name_fullpath):
    def checkMagic(self, file_name_fullpath):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Appcompat_Raw_hive(Ingest):
    def __init__(self):
    def getHostName(self, file_name_fullpath):
    def checkMagic(self, file_name_fullpath):
    def calculateID(self, file_name_fullpath):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Appcompat_miracquisition(Ingest):
    def __init__(self):
    def getHostName(self, file_name_fullpath):
    def checkMagic(self, file_name_fullpath):
    def calculateID(self, file_name_fullpath):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Amcache_Raw_hive(Ingest):
    def __init__(self):
    def checkMagic(self, file_name_fullpath):
    def calculateID(self, file_name_fullpath):
    def getHostName(self, file_name_fullpath):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData): class Appcompat_mirShimShady_v1(Ingest):
    def __init__(self):
    def checkMagic(self, file_name_fullpath):
    def calculateID(self, file_name_fullpath):
    def _processElement(self, element, tag_dict, tag_prefix = ""):
    def processFile(self, file_fullpath, hostID, instanceID, rowsData):
import logging
import Queue
import os
import re
import sqlite3
import ntpath
import time
import struct
import appDB
import settings
import zipfile
import sys
import traceback
import gc
import cProfile
import json
    import pyregf
from mpEngineProdCons import MPEngineProdCons
from mpEngineWorker import MPEngineWorker
from contextlib import closing
from appAux import update_progress, chunks, loadFile, psutil_phymem_usage, file_size
from ShimCacheParser_ACP import read_mir, write_it
from AmCacheParser import _processAmCacheFile_StringIO
from datetime import timedelta, datetime
from Ingest import issues_document
from Ingest import appcompat_hxregistryaudit
from Ingest import appcompat_parsed
from Ingest import appcompat_mirregistryaudit
from Ingest import amcache_miracquisition
from Ingest import appcompat_mirlua_v1
from Ingest import appcompat_mirlua_v2
from Ingest import amcache_mirlua_v1
from Ingest import appcompat_csv
from Ingest import appcompat_redline
from Ingest import appcompat_raw_hive
from Ingest import appcompat_miracquisition
from Ingest import amcache_raw_hive
from Ingest import appcompat_mirShimShady_v1
            # Check if Host exists
            c.execute("SELECT count(*) FROM Hosts WHERE HostName = '%s'" % hostName)
            data = c.fetchone()[0]
            if (data != 0):
                # Host already has at least one instance in the DB
                c.execute("SELECT HostID, Instances FROM Hosts WHERE HostName = '%s'" % hostName)
                data = c.fetchone()
                tmpHostID = data[0]
                tmpInstances = eval(data[1])
                for (file_fullpath, ingest_plugin) in hostsTest[hostName]:
                    try:
                        logger.debug("[%s] Grabbing instanceID from file: %s" % (ingest_plugin, file_fullpath))
                        instance_ID = CalculateInstanceID(file_fullpath, ingest_plugin)
                    except Exception:
                        logger.error("Error parsing: %s (skipping)" % file_fullpath)
                        traceback.print_exc(file=sys.stdout)
                    else:
                        if str(instance_ID) not in tmpInstances:
                            tmpInstances.append(str(instance_ID))
                            hostsProcess.append((file_fullpath, instance_ID, hostName, tmpHostID, ingest_plugin))
                        else:
                            logger.debug("Duplicate host and instance found: %s" %hostName)
                            continue
                # Save updated Instances list
                c.execute("UPDATE Hosts SET Instances = %s, InstancesCounter = %d WHERE HostName = '%s'" % ('"' + str(repr(tmpInstances)) + '"', len(tmpInstances), hostName))
            else:
                # Host does not exist. Add instance and grab the host ID.
                tmpInstances = []
                newInstances = []
                for (file_fullpath, ingest_plugin) in hostsTest[hostName]:
                    try:
                        logger.debug("[%s] Grabbing instanceID from file: %s" % (ingest_plugin, file_fullpath))
                        instance_ID = CalculateInstanceID(file_fullpath, ingest_plugin)
                    except Exception:
                        logger.error("Error parsing: %s (skipping)" % file_fullpath)
                        traceback.print_exc(file=sys.stdout)
                    else:
                        if str(instance_ID) not in tmpInstances:
                            tmpInstances.append(str(instance_ID))
                            newInstances.append((file_fullpath, instance_ID, ingest_plugin))

                c.execute("INSERT INTO Hosts VALUES (NULL,%s,%s,%d,%d,%d)" % ('"' + hostName + '"', '"' + str(repr(tmpInstances)) + '"', len(tmpInstances), 0, 0))
                tmpHostID = c.lastrowid
                for (file_fullpath, instance_ID, ingest_plugin) in newInstances:
                    # todo: Do we want/need each row to track from what instance it came?
                    hostsProcess.append((file_fullpath, instance_ID, hostName, tmpHostID, ingest_plugin))
            # Update progress
            progress_current += 1
            if settings.logger_getDebugMode():
                status_extra_data = " [RAM: %d%%]" % psutil_phymem_usage()
            else: status_extra_data = ""
            logger.info(update_progress(min(1, float(progress_current) / float(progress_total)), "Calculate ID's for new hosts/instances%s" % status_extra_data, True))
        conn.commit()

    # Return hosts to be processed
    return hostsProcess


def parseManifestAuditFileName(jsondata, zip_archive_filename):
    # Parse manifest.json data and return files which will need to be processed
    file_list = []
    m = re.match(r'^.*(?:\\|\/)(.*)[-_].{22}(-[0-9]+-[0-9]+){0,1}\..{3}$', zip_archive_filename)
    if m:
        hostname = m.group(1)
        data = json.load(jsondata)
        if 'audits' in data:
            for audit in data['audits']:
                if 'sysinfo' in audit['generator']: continue
                if 'install' not in audit['generator']:

                    if 'registry-api' in audit['generator'] or 'w32registryapi' in audit['generator']:
                        for result in audit['results']:
                            if 'application/xml' in result['type']:
                                file_list.append((os.path.join(zip_archive_filename, result['payload']), os.path.join(zip_archive_filename, hostname + "_" + result['payload'] + ".xml")))
                            else: continue
                    elif 'plugin-execute' in audit['generator']:
                        for result in audit['results']:
                            if 'application/vnd.mandiant.issues+xml' not in result['type']:
                                file_list.append((os.path.join(zip_archive_filename, result['payload']), os.path.join(zip_archive_filename, hostname + "_" + result['payload'] + ".xml")))
                            else: continue
                    elif 'w32scripting-persistence' in audit['generator']:
                        for result in audit['results']:
                            if 'application/vnd.mandiant.issues+xml' not in result['type']:
                                file_list.append((os.path.join(zip_archive_filename, result['payload']), os.path.join(zip_archive_filename, hostname + "_" + result['payload'] + ".xml")))
                            else: continue
                    elif 'file-acquisition' in audit['generator']:
                        for result in audit['results']:
                            if 'application/vnd.mandiant.issues+xml' not in result['type']:
                                file_list.append((os.path.join(zip_archive_filename, result['payload']), os.path.join(zip_archive_filename, hostname + "_" + result['payload'] + ".xml")))
                            else: continue
                    elif 'AppCompatCache' in audit['generator']:
                        for result in audit['results']:
                            if 'application/vnd.mandiant.issues+xml' not in result['type']:
                                file_list.append((os.path.join(zip_archive_filename, result['payload']), os.path.join(zip_archive_filename, hostname + "_" + result['payload'] + ".xml")))
                            else: continue
        else:
            logger.warning("HX script execution failed for host: %s, ignoring" % hostname)
    else:
        logger.error("Unable to extract hostname on parseManifestAuditFileName: %s" % zip_archive_filename)

    if len(file_list) == 0:
        logger.warning("No file that could be processed found on manifest.json (likely to be a failed script run) for: %s [%d bytes]" % (zip_archive_filename, file_size(zip_archive_filename)))
    return file_list

def processArchives(filename, file_filter):
    # Process zip file if required and return a list of files to process
    files_to_process = []

    if filename.endswith('.zip'):
        try:
            zip_archive_filename = filename
            # Open the zip archive:
            zip_archive = zipfile.ZipFile(loadFile(zip_archive_filename), "r")
            zipFileList = zip_archive.namelist()
            countTotalFiles = len(zipFileList)
            logger.info("Total files in %s: %d" % (zip_archive_filename, countTotalFiles))
            logger.info("Hold on while we check the zipped files...")

            # Check if it's an HX audit zip file:
            if 'manifest.json' in zipFileList:

                jsondata = loadFile(os.path.join(zip_archive_filename, 'manifest.json'))