VIRUS_TOTAL = "http://www.virustotal.com/en/file/%s/analysis" def get_csv_writer(csvfile):
    return UnicodeWriter(csvfile, quoting=csv.QUOTE_ALL)
 def get_json_writer(json_file):
    return UnicodeJsonDumper(json_file)
 def write_to_csv(arr_data, csv_writer):
    """Writes contents to a CSV file using UTF-8"""
    csv_writer.writerow(arr_data)
 def write_to_json(header, arr_data, json_writer):
    """Write data in json files using UTF-8"""
    try:
        json_writer.dump_json({
            h: arr_data[index] for index, h in enumerate(header)
        })
    except IndexError:
        json_writer.dump_json({
            header[index]: d for index, d in enumerate(arr_data)
        })
 def close_json_writer(json_writer):
    json_writer.close_current()
 def process_size(size_str):
    unities = {"k": 1024L, "M": 1024L * 1024L, "G": 1024L * 1024L * 1024L}
    suffix = size_str[len(size_str) - 1:]
    value = size_str[:len(size_str) - 1]
    return long(value) * unities[suffix]
 def record_sha256_logs(fr, fw):
    with open(fw, "a") as hash_file:
        m = process_sha256(fr)
        hash_file.write(fr + "," + m + "\n")
        hash_file.close()
 def process_hashes(path):
    content = open(path, "rb").read()
    return hashlib.md5(content).hexdigest(), hashlib.sha1(content).hexdigest(), hashlib.sha256(content).hexdigest()
 class _PE(_Base_Modules):
    def __init__(self, path, params):
        super(_PE, self).__init__(path, params)
        self.hdll = windll.checksignfromcat
        self.beginfunc = self.hdll.BeginProcessFile
        self.beginfunc.argtypes = [c_wchar_p]
        self.beginfunc.restype = c_int
        self.countfunc = self.hdll.GetCertificatesCount
        self.countfunc.argtypes = [c_int]
        self.countfunc.restype = c_int
        self.issuerfunc = self.hdll.GetCertificateIssuer
        self.issuerfunc.argtypes = [c_int, c_int]
        self.issuerfunc.restype = POINTER(c_wchar_p)
        self.subjectfunc = self.hdll.GetCertificateSubject
        self.subjectfunc.argtypes = [c_int, c_int]
        self.subjectfunc.restype = POINTER(c_wchar_p)
        self.endfunc = self.hdll.EndProcessFile
        self.endfunc.argtypes = [c_int]
        self.endfunc.restype = c_int
        self.cert_filtered_issuer = self.params['cert_filtered_issuer'].split('|')
        self.cert_filtered_subject = self.params['cert_filtered_subject'].split('|')

    def process(self):
        return self.filtered_certificates()

    def _imphash(self):
        pass

    def _pehash(self):
        pass

    def get_certificates(self):
        # pe = PeParser(self.path)
        # issuer_value = ''
        # subject_value = ''
        # if pe.signature:
        #     issuer_value = pe.signature.getissuer()
        #     subject_value = pe.signature.getsubject()
        # else:
        #     pe.unmap()
        #     return None, None
        # return issuer_value,subject_value

        issuer_value = ''
        subject_value = ''

        handle = 0
        try:
            try:
                handle = self.beginfunc(self.path)

                if handle != 0:
                    count = self.countfunc(handle)
                    for index in xrange(count):
                        issuer = self.issuerfunc(handle, index)
                        subject = self.subjectfunc(handle, index)
                        issuer_value += ' %s' % wstring_at(issuer)
                        subject_value += ' %s' % wstring_at(subject)

            except WindowsError as e:
                exc_type, exc_obj, exc_tb = sys.exc_info()
                self.params['logger'].error('Windows Error parsing with Catalog: %s line %s' % (self.path, exc_tb.tb_lineno))
                self.params['logger'].error(str(e))
                self.endfunc(handle)
            if len(issuer_value) == 0 and len(subject_value) == 0:
                try:
                    pe = PeParser(self.path)

                    issuer_value = ''
                    subject_value = ''
                    if pe.signature:
                        issuer_value = pe.signature.getissuer()
                        subject_value = pe.signature.getsubject()
                        pe.unmap()
                        #print issuer_value
                        #print subject_value
                    else:
                        pe.unmap()
                        return None, None
                except WindowsError as e:
                    exc_type, exc_obj, exc_tb = sys.exc_info()
                    self.params['logger'].error('Windows Error parsing with pe-tools: %s line %s' % (self.path, exc_tb.tb_lineno))
                    self.params['logger'].error(str(e))
                    return None, None

        except WindowsError as e:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            self.params['logger'].error('Windows Error parsing: %s line %s' % (self.path, exc_tb.tb_lineno))
            self.params['logger'].error(str(e))
            return None, None
        except Exception as e:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            self.params['logger'].error('Error parsing: %s line %s' % (self.path, exc_tb.tb_lineno))
            self.params['logger'].error(str(e))
            return None, None

        return issuer_value, subject_value

    def filtered_certificates(self):
        issuer, subject = self.get_certificates()
        self.params['logger'].debug('Issuer: %s Subject: %s' % (issuer,subject))
        if issuer and subject:
            for i in self.cert_filtered_issuer:
                if i in issuer:
                    return True
            for s in self.cert_filtered_subject:
                if s in subject:
                    return True
        else:
            return False

import datetime
import os
import yaml
from filecatcher.listfiles import _ListFiles
from settings import VIRUS_TOTAL
from utils.utils import get_csv_writer, get_json_writer, write_to_csv, write_to_json, close_json_writer, process_size,\
     record_sha256_logs, process_hashes
from utils.utils_rawstring import sekoiamagic
from filecatcher.modules.PE import _PE
from filecatcher.modules.intel import _Intel
from archives import _Archives
                else:
                    try:
                        true_f = os.path.splitdrive(self.systemroot)[0] + '\\' + f.split('\\', 6)[-1]
                        self.logger.warn('file %s not cached by size %s or date %s' % (true_f, os.stat(f).st_size,
                                                                                       self._get_modification_date(f)))
                    except Exception as e:
                        self.logger.error(e)
                        self.logger.error(f)

    @staticmethod
    def _get_creation_date(f):
        t = os.path.getctime(f)
        date_value = datetime.datetime.fromtimestamp(t)
        return date_value

    @staticmethod
    def _get_modification_date(f):
        t = os.path.getmtime(f)
        date_value = datetime.datetime.fromtimestamp(t)
        return date_value

    def _process_hashes(self, f):
        if hasattr(self, 'vss') and self.vss:
            return self.vss.process_hash_value(f)
        else:
            return process_hashes(f)

    def _filtered_by_date(self, f):
        if self.limit_days == 'unlimited':
            return True
        try:

            return (datetime.datetime.now() - self._get_modification_date(f)).days < int(self.limit_days)
        except Exception as e:
            self.logger.error(e)
            return True

    def _filtered_magic(self, f):
        try:
            mime = sekoiamagic(f)
            wildcard_mime_filter = mime in self.mime_filter
            wildcard_mime_zip = mime in self.mime_zip
            if "*" in self.mime_filter:
                wildcard_mime_filter = True
            if "*" in self.mime_zip:
                wildcard_mime_zip = True
            return wildcard_mime_filter, wildcard_mime_zip, mime
        except Exception as e:
            self.logger.error(e)
            return False, False, None

    def _filtered_size(self, f):
        try:
            return self.size_min <= os.stat(f).st_size <= self.size_max  # and self.size_max > os.stat(f).st_size
        except Exception as e:
            self.logger.error(e)
            return False

    @staticmethod
    def _filtered_ext(ext, filters):
        if '|EMPTY|' in filters:
            if len(ext) == 0:
                return True
        if '*' in filters:
            return True
        if 'all' in filters:
            return True
        else:
            return ext in filters

    def _filtered(self, items):
        if self.compare == 'AND':
            return all(items)
        if self.compare == 'OR':
            return any(items)
        return True

    def _is_PE(self, mime):
        if mime:
            return mime in self.params['pe_mime_type']
        else:
            return False

    @staticmethod
    def _check_depth(f, depth, directory):
        if depth == '*':
            return True
        return int(depth) >= f[len(directory) + len(os.path.sep):].count(os.path.sep)

    def _get_url_VT(self, sha256):
        if 'N/A' in sha256:
            url_vt = 'not URL VT'
        else:
            url_vt = unicode(VIRUS_TOTAL % sha256)
        return url_vt

    def _csv_infos_fs(self, files):
        with open(self.output_dir + '\\' + self.computer_name + '_Filecatcher' + self.rand_ext, 'wb') as fw:
            csv_writer = get_csv_writer(fw)
            write_to_csv(['COMPUTER NAME', 'TYPE', 'DATE', 'PATH', 'MD5', 'SHA1', 'SHA256', 'MIMETYPE', 'ZIP',
                          'EMPTY', 'VT'], csv_writer)
            for f, mime, md5, sha1, sha256, zip_value, datem, empty in files:
                f = os.path.splitdrive(self.systemroot)[0] + '\\' + f.split('\\', 6)[-1]
                write_to_csv([self.computer_name, 'Filecatcher', unicode(datem),
                              unicode(f), unicode(md5), unicode(sha1), unicode(sha256), unicode(mime),
                              unicode(zip_value), unicode(empty), self._get_url_VT(sha256)], csv_writer)
        record_sha256_logs(self.output_dir + '\\' + self.computer_name + '_Filecatcher' + self.rand_ext,
                           self.output_dir + '\\' + self.computer_name + '_sha256.log')

        if self.zip_file:
            self.zip_file.close()

    def _json_infos_fs(self, files):
        if self.destination == 'local':
            with open(self.output_dir + '\\' + self.computer_name + '_Filecatcher' + self.rand_ext, 'wb') as fw:
                json_writer = get_json_writer(fw)
                headers = ['COMPUTER NAME', 'TYPE', 'DATE', 'PATH', 'MD5', 'SHA1', 'SHA256', 'MIMETYPE', 'ZIP',
                           'EMPTY', 'VT']
                for f, mime, md5, sha1, sha256, zip_value, datem, empty in files:
                    f = os.path.splitdrive(self.systemroot)[0] + '\\' + f.split('\\', 6)[-1]

                    write_to_json(headers, [self.computer_name, 'Filecatcher', unicode(datem), unicode(f), unicode(md5),