#!/usr/bin/env python
'''
Main AVClass class
'''

import re
import string
from collections import OrderedDict as OrdDict
from collections import namedtuple
from operator import itemgetter, attrgetter
import statistics

SampleInfo = namedtuple('SampleInfo', 
                        ['md5', 'sha1', 'sha256', 'labels'])

eng_dict = 0
non_dga_score_dict = {}

class AvLabels:
    '''
    Class to operate on AV labels, 
    such as extracting the most likely family name.
    '''
    def __init__(self, gen_file = None, alias_file = None, av_file = None):

        # Read generic token set from file
        self.gen_set = self.read_generics(gen_file) if gen_file else set()

        # Read aliases map from file
        self.aliases_map = self.read_aliases(alias_file) if alias_file else {}

        # Read AV engine set from file
        self.avs = self.read_avs(av_file) if av_file else None

    @staticmethod
    def read_aliases(alfile):
        '''Read aliases map from given file'''
        if alfile is None:
            return {}
        almap = {}
        with open(alfile, 'r') as fd:
            for line in fd:
                tmp_line = line.strip().split()[0:2]
                if len(tmp_line) == 0:
                    continue
                alias, token = tmp_line
                almap[alias] = token
        return almap

    @staticmethod
    def read_generics(generics_file):
        '''Read generic token set from given file'''
        gen_set = set()
        with open(generics_file) as gen_fd:
            for line in gen_fd:
                if line.startswith('#') or line == '\n':
                    continue
                gen_set.add(line.strip())
        return gen_set

    @staticmethod
    def read_avs(avs_file):
        '''Read AV engine set from given file'''
        with open(avs_file) as fd:
            avs = set(map(str.strip, fd.readlines()))
        return avs

    @staticmethod
    def get_sample_info(vt_rep, from_vt):
        '''Parse and extract sample information from JSON line
           Returns a SampleInfo named tuple: md5, sha1, sha256, label_pairs 
        '''
        if 'results' in vt_rep:
            vt_rep = vt_rep['results']
        label_pairs = []
        if from_vt:
            try:
                scans = vt_rep['scans']
            except KeyError:
                return None
            for av, res in scans.items():
                if res['detected']:
                    label = res['result']
                    clean_label = ''.join(filter(lambda x: x in string.printable, label)).strip()
                    label_pairs.append((av, clean_label))
        else:
            label_pairs = vt_rep['av_labels']

        return SampleInfo(vt_rep['md5'], vt_rep['sha1'], vt_rep['sha256'],
                          label_pairs) 

    @staticmethod
    def is_pup(av_label_pairs):
        '''This function classifies the sample as PUP or not 
           using the AV labels as explained in the paper:
           "Certified PUP: Abuse in Authenticode Code Signing" 
           (ACM CCS 2015)
           It uses the AV labels of 11 specific AVs. 
           The function checks for 13 keywords used to indicate PUP.
           Return:
              True/False/None
        '''
        # If no AV labels, nothing to do, return
        if not av_label_pairs:
            return None
        # Initialize
        pup = False
        threshold = 0.5
        # AVs to use
        av_set = set(['Malwarebytes', 'K7AntiVirus', 'Avast',
                  'AhnLab-V3', 'Kaspersky', 'K7GW', 'Ikarus',
                  'Fortinet', 'Antiy-AVL', 'Agnitum', 'ESET-NOD32'])
        # Tags that indicate PUP
        tags = set(['PUA', 'Adware', 'PUP', 'Unwanted', 'Riskware', 'grayware',
                    'Unwnt', 'Adknowledge', 'toolbar', 'casino', 'casonline',
                    'AdLoad', 'not-a-virus'])

        # Set with (AV name, Flagged/not flagged as PUP), for AVs in av_set
        bool_set = set([(pair[0], t.lower() in pair[1].lower()) for t in tags
                        for pair in av_label_pairs
                        if pair[0] in av_set])

        # Number of AVs that had a label for the sample
        av_detected = len([p[0] for p in av_label_pairs
                           if p[0] in av_set])

        # Number of AVs that flagged the sample as PUP
        av_pup = list(map(lambda x: x[1], bool_set)).count(True)

        # Flag as PUP according to a threshold
        if (float(av_pup) >= float(av_detected)*threshold) and av_pup != 0:
            pup = True
        return pup

    @staticmethod
    def is_dga(token, eng_dict_file = 'data/top10000en.txt', n = 3, threshold = 2):
        '''Return True if a given label is (probably) generated by DGA'''
        label = token[0]
        
        global eng_dict
        if eng_dict == 0:
            with open(eng_dict_file) as f:
                eng_dict = f.readlines()

        def ngrams(sequence, n):
            return list(zip(*(sequence[i:] for i in range(n))))

        def get_meaningful_characters_ratio(label):
            max_len = 0
            if len(label) < 3:
                return 0
            for i in range(n, len(label)):
                tuples = ngrams(label, i)
                tmp_sum = 0
                tmp_s = []

                for s in [''.join(t) for t in tuples]:
                    for words in eng_dict:
                        if s in words:
                            tmp_sum += len(s)
                            tmp_s.append(s)

                if tmp_sum > max_len:
                    max_len = tmp_sum

                return max_len / int(len(label))

        def get_ngram_normality_score(label):
            tmp_sum = 0                        
            tuples = ngrams(label, n)            
            for s in [''.join(t) for t in tuples]:
                for words in eng_dict:
                    if s in words:
                        tmp_sum += 1

            return tmp_sum / (len(label) - n + 1)

        global non_dga_score_dict
        if label in non_dga_score_dict:
            non_dga_score = non_dga_score_dict[label]
        else:
            meaningful_characters_ratio = get_meaningful_characters_ratio(label)
            ngram_normality_score = get_ngram_normality_score(label)
            non_dga_score = statistics.harmonic_mean([meaningful_characters_ratio, ngram_normality_score])
            non_dga_score_dict[label] = non_dga_score    
    
        if non_dga_score < threshold:
            return True
        else:
            return False

    @staticmethod
    def __remove_suffixes(av_name, label):
        '''Remove AV specific suffixes from given label
           Returns updated label'''

        # Truncate after last '.'
        if av_name in set(['Norman', 'Avast', 'Avira', 'Kaspersky',
                          'ESET-NOD32', 'Fortinet', 'Jiangmin', 'Comodo',
                          'GData', 'Avast', 'Sophos',
                          'TrendMicro-HouseCall', 'TrendMicro',
                          'NANO-Antivirus', 'Microsoft']):
            label = label.rsplit('.', 1)[0]

        # Truncate after last '.' 
        # if suffix only contains digits or uppercase (no lowercase) chars
        if av_name == 'AVG':
            tokens = label.rsplit('.', 1)
            if len(tokens) > 1 and re.match("^[A-Z0-9]+$", tokens[1]):
                label = tokens[0]

        # Truncate after last '!'
        if av_name in set(['Agnitum','McAffee','McAffee-GW-Edition']):
            label = label.rsplit('!', 1)[0]

        # Truncate after last '('
        if av_name in set(['K7AntiVirus', 'K7GW']):
            label = label.rsplit('(', 1)[0]

        # Truncate after last '@'
        # GData would belong here, but already trimmed earlier
        if av_name in set(['Ad-Aware', 'BitDefender', 'Emsisoft', 'F-Secure', 
                          'eScan', 'AegisLab']):
            label = label.rsplit('(', 1)[0]

        return label


    def __normalize(self, label, hashes):
        '''Tokenize label, filter tokens, and replace aliases'''

        # If empty label, nothing to do
        if not label:
            return []

        # Initialize list of tokens to return
        ret = []

        # Split label into tokens and process each token
        for token in re.split("[^0-9a-zA-Z]", label):
            # Convert to lowercase
            token = token.lower()

            # Remove digits at the end
            end_len = len(re.findall("\d*$", token)[0])
            if end_len:
                token = token[:-end_len]

            # Ignore short token
            if len(token) < 4:
                continue

            # Remove generic tokens
            if token in self.gen_set:
                continue

            # Ignore token if prefix of a hash of the sample 
            # Most AVs use MD5 prefixes in labels, 
            # but we check SHA1 and SHA256 as well
            hash_token = False
            for hash_str in hashes:
                if hash_str[0:len(token)] == token:
                  hash_token = True
                  break
            if hash_token:
                continue

            # Replace alias
            token = self.aliases_map[token] if token in self.aliases_map \
                                            else token

            # Add token
            ret.append(token)

        return ret

    def get_family_ranking(self, sample_info):
        '''
        Returns sorted dictionary of most likely family names for sample
        '''
        # Extract info from named tuple
        av_label_pairs = sample_info[3]
        hashes = [ sample_info[0], sample_info[1], sample_info[2] ]

        # Whitelist the AVs to filter the ones with meaningful labels
        av_whitelist = self.avs

        # Initialize auxiliary data structures
        labels_seen = set()
        token_map = {}

        # Process each AV label
        for (av_name, label) in av_label_pairs:
            # If empty label, nothing to do
            if not label:
                continue

            ################
            # AV selection #
            ################
            if av_whitelist and av_name not in av_whitelist:
                continue

            #####################
            # Duplicate removal #
            #####################

            # If label ends in ' (B)', remove it
            if label.endswith(' (B)'):
                label = label[:-4]

            # If we have seen the label before, skip
            if label in labels_seen:
                continue
            # If not, we add it to the set of labels seen
            else:
                labels_seen.add(label)

            ##################
            # Suffix removal #
            ##################
            label = self.__remove_suffixes(av_name, label)

            ########################################################
            # Tokenization, token filtering, and alias replacement #
            ########################################################
            tokens = self.__normalize(label, hashes)

            # Increase token count in map
            for t in tokens:
                c = token_map[t] if t in token_map else 0
                token_map[t] = c + 1

        ##################################################################
        # Token ranking: sorts tokens by decreasing count and then token #
        ##################################################################
        sorted_tokens = sorted(token_map.items(), 
                                key=itemgetter(1,0), 
                                reverse=True)

        # Delete the tokens appearing only in one AV, add rest to output
        sorted_dict = OrdDict()
        for t, c in sorted_tokens:
            if c > 1:
                sorted_dict[t] = c
            else:
                break
        
        return sorted_dict


#!/usr/bin/env python
'''
Main AVClass class
'''

import re
import string
from collections import OrderedDict as OrdDict
from collections import namedtuple
from operator import itemgetter, attrgetter
import statistics

SampleInfo = namedtuple('SampleInfo', 
                        ['md5', 'sha1', 'sha256', 'labels'])

eng_dict = 0
non_dga_score_dict = {}

class AvLabels:
    '''
    Class to operate on AV labels, 
    such as extracting the most likely family name.
    '''
    def __init__(self, gen_file = None, alias_file = None, av_file = None):

        # Read generic token set from file
        self.gen_set = self.read_generics(gen_file) if gen_file else set()

        # Read aliases map from file
        self.aliases_map = self.read_aliases(alias_file) if alias_file else {}

        # Read AV engine set from file
        self.avs = self.read_avs(av_file) if av_file else None

    @staticmethod
    def read_aliases(alfile):
        '''Read aliases map from given file'''
        if alfile is None:
            return {}
        almap = {}
        with open(alfile, 'r') as fd:
            for line in fd:
                tmp_line = line.strip().split()[0:2]
                if len(tmp_line) == 0:
                    continue
                alias, token = tmp_line
                almap[alias] = token
        return almap

    @staticmethod
    def read_generics(generics_file):
        '''Read generic token set from given file'''
        gen_set = set()
        with open(generics_file) as gen_fd:
            for line in gen_fd:
                if line.startswith('#') or line == '\n':
                    continue
                gen_set.add(line.strip())
        return gen_set

    @staticmethod
    def read_avs(avs_file):
        '''Read AV engine set from given file'''
        with open(avs_file) as fd:
            avs = set(map(str.strip, fd.readlines()))
        return avs

    @staticmethod
    def get_sample_info(vt_rep, from_vt):
        '''Parse and extract sample information from JSON line
           Returns a SampleInfo named tuple: md5, sha1, sha256, label_pairs 
        '''
        if 'results' in vt_rep:
            vt_rep = vt_rep['results']
        label_pairs = []
        if from_vt:
            try:
                scans = vt_rep['scans']
            except KeyError:
                return None
            for av, res in scans.items():
                if res['detected']:
                    label = res['result']
                    clean_label = ''.join(filter(lambda x: x in string.printable, label)).strip()
                    label_pairs.append((av, clean_label))
        else:
            label_pairs = vt_rep['av_labels']

        return SampleInfo(vt_rep['md5'], vt_rep['sha1'], vt_rep['sha256'],
                          label_pairs) 

    @staticmethod
    def is_pup(av_label_pairs):
        '''This function classifies the sample as PUP or not 
           using the AV labels as explained in the paper:
           "Certified PUP: Abuse in Authenticode Code Signing" 
           (ACM CCS 2015)
           It uses the AV labels of 11 specific AVs. 
           The function checks for 13 keywords used to indicate PUP.
           Return:
              True/False/None
        '''
        # If no AV labels, nothing to do, return
        if not av_label_pairs:
            return None
        # Initialize
        pup = False
        threshold = 0.5
        # AVs to use
        av_set = set(['Malwarebytes', 'K7AntiVirus', 'Avast',
                  'AhnLab-V3', 'Kaspersky', 'K7GW', 'Ikarus',
                  'Fortinet', 'Antiy-AVL', 'Agnitum', 'ESET-NOD32'])
        # Tags that indicate PUP
        tags = set(['PUA', 'Adware', 'PUP', 'Unwanted', 'Riskware', 'grayware',
                    'Unwnt', 'Adknowledge', 'toolbar', 'casino', 'casonline',
                    'AdLoad', 'not-a-virus'])

        # Set with (AV name, Flagged/not flagged as PUP), for AVs in av_set
        bool_set = set([(pair[0], t.lower() in pair[1].lower()) for t in tags
                        for pair in av_label_pairs
                        if pair[0] in av_set])

        # Number of AVs that had a label for the sample
        av_detected = len([p[0] for p in av_label_pairs
                           if p[0] in av_set])

        # Number of AVs that flagged the sample as PUP
        av_pup = list(map(lambda x: x[1], bool_set)).count(True)

        # Flag as PUP according to a threshold
        if (float(av_pup) >= float(av_detected)*threshold) and av_pup != 0:
            pup = True
        return pup

    @staticmethod
    def is_dga(token, eng_dict_file = 'data/top10000en.txt', n = 3, threshold = 2):
        '''Return True if a given label is (probably) generated by DGA'''
        label = token[0]
        
        global eng_dict
        if eng_dict == 0:
            with open(eng_dict_file) as f:
                eng_dict = f.readlines()

        def ngrams(sequence, n):
            return list(zip(*(sequence[i:] for i in range(n))))

        def get_meaningful_characters_ratio(label):
            max_len = 0
            if len(label) < 3:
                return 0
            for i in range(n, len(label)):
                tuples = ngrams(label, i)
                tmp_sum = 0
                tmp_s = []

                for s in [''.join(t) for t in tuples]:
                    for words in eng_dict:
                        if s in words:
                            tmp_sum += len(s)
                            tmp_s.append(s)

                if tmp_sum > max_len:
                    max_len = tmp_sum

                return max_len / int(len(label))

        def get_ngram_normality_score(label):
            tmp_sum = 0                        
            tuples = ngrams(label, n)            
            for s in [''.join(t) for t in tuples]:
                for words in eng_dict:
                    if s in words:
                        tmp_sum += 1

            return tmp_sum / (len(label) - n + 1)

        global non_dga_score_dict
        if label in non_dga_score_dict:
            non_dga_score = non_dga_score_dict[label]
        else:
            meaningful_characters_ratio = get_meaningful_characters_ratio(label)
            ngram_normality_score = get_ngram_normality_score(label)
            non_dga_score = statistics.harmonic_mean([meaningful_characters_ratio, ngram_normality_score])
            non_dga_score_dict[label] = non_dga_score    
    
        if non_dga_score < threshold:
            return True
        else:
            return False

    @staticmethod
    def __remove_suffixes(av_name, label):
        '''Remove AV specific suffixes from given label
           Returns updated label'''

        # Truncate after last '.'
        if av_name in set(['Norman', 'Avast', 'Avira', 'Kaspersky',
                          'ESET-NOD32', 'Fortinet', 'Jiangmin', 'Comodo',
                          'GData', 'Avast', 'Sophos',
                          'TrendMicro-HouseCall', 'TrendMicro',
                          'NANO-Antivirus', 'Microsoft']):
            label = label.rsplit('.', 1)[0]

        # Truncate after last '.' 
        # if suffix only contains digits or uppercase (no lowercase) chars
        if av_name == 'AVG':
            tokens = label.rsplit('.', 1)
            if len(tokens) > 1 and re.match("^[A-Z0-9]+$", tokens[1]):
                label = tokens[0]

        # Truncate after last '!'
        if av_name in set(['Agnitum','McAffee','McAffee-GW-Edition']):
            label = label.rsplit('!', 1)[0]

        # Truncate after last '('
        if av_name in set(['K7AntiVirus', 'K7GW']):
            label = label.rsplit('(', 1)[0]

        # Truncate after last '@'
        # GData would belong here, but already trimmed earlier
        if av_name in set(['Ad-Aware', 'BitDefender', 'Emsisoft', 'F-Secure', 
                          'eScan', 'AegisLab']):
            label = label.rsplit('(', 1)[0]

        return label


    def __normalize(self, label, hashes):
        '''Tokenize label, filter tokens, and replace aliases'''

        # If empty label, nothing to do
        if not label:
            return []

        # Initialize list of tokens to return
        ret = []

        # Split label into tokens and process each token
        for token in re.split("[^0-9a-zA-Z]", label):
            # Convert to lowercase
            token = token.lower()

            # Remove digits at the end
            end_len = len(re.findall("\d*$", token)[0])
            if end_len:
                token = token[:-end_len]

            # Ignore short token
            if len(token) < 4:
                continue

            # Remove generic tokens
            if token in self.gen_set:
                continue

            # Ignore token if prefix of a hash of the sample 
            # Most AVs use MD5 prefixes in labels, 
            # but we check SHA1 and SHA256 as well
            hash_token = False
            for hash_str in hashes:
                if hash_str[0:len(token)] == token:
                  hash_token = True
                  break
            if hash_token:
                continue

            # Replace alias
            token = self.aliases_map[token] if token in self.aliases_map \
                                            else token

            # Add token
            ret.append(token)

        return ret

    def get_family_ranking(self, sample_info):
        '''
        Returns sorted dictionary of most likely family names for sample
        '''
        # Extract info from named tuple
        av_label_pairs = sample_info[3]
        hashes = [ sample_info[0], sample_info[1], sample_info[2] ]

        # Whitelist the AVs to filter the ones with meaningful labels
        av_whitelist = self.avs

        # Initialize auxiliary data structures
        labels_seen = set()
        token_map = {}

        # Process each AV label
        for (av_name, label) in av_label_pairs:
            # If empty label, nothing to do
            if not label:
                continue

            ################
            # AV selection #
            ################
            if av_whitelist and av_name not in av_whitelist:
                continue

            #####################
            # Duplicate removal #
            #####################

            # If label ends in ' (B)', remove it
            if label.endswith(' (B)'):
                label = label[:-4]

            # If we have seen the label before, skip
            if label in labels_seen:
                continue
            # If not, we add it to the set of labels seen
            else:
                labels_seen.add(label)

            ##################
            # Suffix removal #
            ##################
            label = self.__remove_suffixes(av_name, label)

            ########################################################
            # Tokenization, token filtering, and alias replacement #
            ########################################################
            tokens = self.__normalize(label, hashes)

            # Increase token count in map
            for t in tokens:
                c = token_map[t] if t in token_map else 0
                token_map[t] = c + 1

        ##################################################################
        # Token ranking: sorts tokens by decreasing count and then token #
        ##################################################################
        sorted_tokens = sorted(token_map.items(), 
                                key=itemgetter(1,0), 
                                reverse=True)

        # Delete the tokens appearing only in one AV, add rest to output
        sorted_dict = OrdDict()
        for t, c in sorted_tokens:
            if c > 1:
                sorted_dict[t] = c
            else:
                break
        
        return sorted_dict


#!/usr/bin/env python Main AVClass class Class to operate on AV labels,      such as extracting the most likely family name. # Read generic token set from file # Read aliases map from file # Read AV engine set from file Read aliases map from given file Read generic token set from given file Read AV engine set from given file Parse and extract sample information from JSON line            Returns a SampleInfo named tuple: md5, sha1, sha256, label_pairs This function classifies the sample as PUP or not             using the AV labels as explained in the paper:            "Certified PUP: Abuse in Authenticode Code Signing"             (ACM CCS 2015)            It uses the AV labels of 11 specific AVs.             The function checks for 13 keywords used to indicate PUP.            Return:               True/False/None # If no AV labels, nothing to do, return # Initialize # AVs to use # Tags that indicate PUP # Set with (AV name, Flagged/not flagged as PUP), for AVs in av_set # Number of AVs that had a label for the sample # Number of AVs that flagged the sample as PUP # Flag as PUP according to a threshold Return True if a given label is (probably) generated by DGA Remove AV specific suffixes from given label            Returns updated label # Truncate after last '.' # Truncate after last '.' # if suffix only contains digits or uppercase (no lowercase) chars # Truncate after last '!' # Truncate after last '(' # Truncate after last '@' # GData would belong here, but already trimmed earlier Tokenize label, filter tokens, and replace aliases # If empty label, nothing to do # Initialize list of tokens to return # Split label into tokens and process each token # Convert to lowercase # Remove digits at the end # Ignore short token # Remove generic tokens # Ignore token if prefix of a hash of the sample # Most AVs use MD5 prefixes in labels, # but we check SHA1 and SHA256 as well # Replace alias # Add token Returns sorted dictionary of most likely family names for sample # Extract info from named tuple # Whitelist the AVs to filter the ones with meaningful labels # Initialize auxiliary data structures # Process each AV label # If empty label, nothing to do ################ # AV selection # ################ ##################### # Duplicate removal # ##################### # If label ends in ' (B)', remove it # If we have seen the label before, skip # If not, we add it to the set of labels seen ################## # Suffix removal # ################## ######################################################## # Tokenization, token filtering, and alias replacement # ######################################################## # Increase token count in map ################################################################## # Token ranking: sorts tokens by decreasing count and then token # ################################################################## # Delete the tokens appearing only in one AV, add rest to output