import json
import pandas as pd
import re
import string
from re import finditer
from bs4 import BeautifulSoup
from tqdm import tqdm


class cleanFormat(object):

    def splitFirstSentence(self, text):
        clean_pattern1 = re.compile('([\s/*=-]+)$|^([\s/*=-]+)')
        clean_pattern2 = re.compile('^([\s*]+)')

        def func(t):
            t = t.strip().replace('&nbsp;', ' ')
            return re.sub(clean_pattern2, '', t).strip()

        text_list = [func(t) for t in re.sub(clean_pattern1, '', text).split('\n')]

        pattern1 = re.compile(r'([^\n]+?)(((?<!i\.e)(?<!e\.g)(?<!\beg)(?<!\bie))\.\s|\.$)')
        pattern2 = re.compile(r'^[A-Z@]|(\(?i\.e\.)|(\(?e\.g\.)|(\(?eg\.)|(\(?ie\.)[\s\S]+')

        start = 0
        for i in range(len(text_list)):
            if text_list[i] == '':
                start = i + 1
            else:
                break

        comment = ''
        for idx, text in enumerate(text_list[start:]):
            if comment == '':
                obj = re.match(pattern1, text)
                if obj:
                    return obj[0]
                else:
                    comment += text
            else:
                if text == '' or re.match(pattern2, text) is not None:
                    return comment
                else:
                    obj = re.match(pattern1, text)
                    if obj:
                        return comment + ' ' + obj[0]
                    else:
                        comment = comment + ' ' + text

        return comment.strip()

    def isEnglish(self, text):
        return text.isascii()

    def findExistComment(self, rawCode):
        rawCode = rawCode.strip()
        p0 = re.compile('^(//|/\*)')
        p1 = re.compile(';\n*(//|/\*)')
        text_list = [t.strip() for t in rawCode.split('\n')]
        for row in text_list:
            if row != '' and (p0.search(row) or p1.search(row)):
                return True
        return False

    def ifALLCommentedOut(self, rawCode):
        rawCode = rawCode.strip()
        p0 = re.compile('(^/\*)')
        if p0.search(rawCode):
            # print(rawCode)
            return True
        p1 = re.compile('(^//)')
        text_list = [t.strip() for t in rawCode.split('\n')]
        for row in text_list:
            if row != '' and not p1.search(row):
                return False
        return True

    def cleanCommentedCode(self, rawCode):
        p = re.compile('(^//)')
        text_list = [t.strip() for t in rawCode.strip().split('\n')]
        new_list = []
        for row in text_list:
            if row != '' and not p.search(row):
                new_list.append(row)
        return '\n'.join(new_list)

    def ifJavaTag(self, text):
        javaTag_pattern1 = re.compile('\{@')  # re.compile('\{@[^\s]+\s[^}]+\}')
        javaTag_pattern2 = re.compile(
            '@(see|throws|link|code|inheritdoc|return|param|author|version|exception|deprecated)')
        if javaTag_pattern1.search(text) or javaTag_pattern2.search(text):
            return True
        else:
            return False

    def findCommonFunc(self, rawCode, language):
        text_list = [t.strip() for t in rawCode.strip().split('\n')]
        text = text_list[0]
        pattern1 = re.compile(' toString\(')
        pattern2 = re.compile('([ _]test[A-Z_(])|([a-z_ ]Test[A-Z_(])')
        pattern3 = re.compile(' [a-zA-Z_]*Constructor[s]?\(')
        pattern5 = re.compile(' (set|get)[A-Z][a-zA-Z_]+\(')

        pattern6 = re.compile('__str__')
        if language == 'java':
            if pattern1.search(text) or pattern2.search(text) or pattern3.search(text):
                return True
            elif pattern5.search(text) and len(text_list) <= 3:
                return True
            else:
                return False
        elif language == 'python':
            if pattern2.search(text) or pattern6.search(text):
                return True
            else:
                return False
        else:
            return False

    def findDeadCode(self, rawCode, language):

        if language == 'java':
            pattern = re.compile('(?!\{\s+\})\{[\s\S]+\}')
            if pattern.search(rawCode):
                return False
            else:
                return True
        else:
            count = 0
            text_list = [t.strip() for t in rawCode.strip().split('\n')]
            for row in text_list:
                if row != '':
                    count += 1
            if count >= 3:
                return False
            else:
                return True

    def shortComment(self, text):
        if len(text.strip().split()) < 3:
            return True
        else:
            return False

    def findPunctuation(self, text):
        p = re.compile('[a-zA-Z]')
        if p.search(text):
            return False
        else:
            return True

    def findInterrogation(self, text):
        url_pattern = re.compile(
            '(http|ftp|https):\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])')
        text = re.sub(url_pattern, '', text)
        pattern1 = re.compile(r'(?i)^(why\b|how\b|what[\']?[s]?\b|where\b|is\b|are\b)')
        pattern2 = re.compile(r'(?i)(\bhow to\b|\bwhy\b|\bwhat (is|was|are)\b)')
        if '?' in text:
            return True
        elif pattern1.search(text) or pattern2.search(text):
            return True
        else:
            return False

    def findUrl(self, text):
        html_pattern = re.compile(
            '(http|ftp|https):\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])')
        if html_pattern.search(text):
            return True
        else:
            return False

    def clean_code_format(self, raw_code, language):
        if language == 'java':
            import javalang
            try:
                tokens = list(javalang.tokenizer.tokenize(raw_code))
                tks_list = []
                for tk in tokens:
                    tks_list.append(tk.value)

                code = " ".join(tks_list)
                return code
            except javalang.tokenizer.LexerError:
                return ''

    def findFirstSentence(self, tokenized_comment, process_comment):
        def cleanPunc(text):
            text = text.translate(str.maketrans(string.punctuation, ' ' * len(string.punctuation)))
            text = re.sub(r'\s+', ' ', text)
            return text.lower().strip()

        rawraw = cleanPunc(BeautifulSoup(tokenized_comment, "html.parser").get_text())
        processprocess = cleanPunc(BeautifulSoup(process_comment, "html.parser").get_text())
        if rawraw == processprocess:
            return False
        else:
            if len(rawraw.split()) > len(processprocess.split()):
                return 'more'
            else:
                return 'less'

    def findAutoGenerated(self, text, raw_text=None):
        auto_generated_pattern1 = re.compile(r'(?i)@[a-zA-Z]*generated\b')
        auto_generated_pattern2 = re.compile('(?i)^([aA]uto[-\s]generated)')
        auto_generated_pattern3 = re.compile('(?i)^(This method initializes)')
        auto_generated_pattern4 = re.compile('(?i)^(This method was generated by)')
        if raw_text is not None:
            if auto_generated_pattern1.search(raw_text):
                return True

        if auto_generated_pattern2.search(text) or auto_generated_pattern3.search(
                text) or auto_generated_pattern4.search(text):
            return True
        else:
            return False


if __name__ == '__main__':
    base_path = './../../../../Downloads/code2comment_dataset/dataset/CodeSearchNet_java_dataset/CodeSearchNet/java/'

    total_num = 0
    total_changedComment_num = 0
    total_code_change_num = 0
    total_comment_change_num = 0
    total_deadCode_num, total_commonFunc_num, total_nonEnglish_num, total_firstSentence_num, total_html_num, total_javaTag_num, total_interrogation_num = 0, 0, 0, 0, 0, 0, 0
    total_partialCommented_num, total_url_num, total_punctuation_num, total_all_commented_code, total_more_num, total_less_num = 0, 0, 0, 0, 0, 0

    code_dict = {}
    unchanged_list = []
    unchanged_list_code = []
    formatFilter = cleanFormat()
    for file in ['train', 'valid', 'test']:
        path = base_path + f'{file}/java_{file}_0.jsonl'
        with open(path, 'r') as f:
            lines = f.readlines()

        if file == 'train':
            for idx in range(1, 16):
                with open(base_path + f'{file}/java_{file}_{idx}.jsonl', 'r') as f:
                    lines += f.readlines()

        file_number = len(lines)
        changedComment_num = 0
        code_change_num = 0
        comment_change_num = 0
        deadCode_num, commonFunc_num, nonEnglish_num, firstSentence_num, html_num, javaTag_num, interrogation_num = 0, 0, 0, 0, 0, 0, 0
        partialCommented_num, url_num, punctuation_num, all_commented_code, more_num, less_num = 0, 0, 0, 0, 0, 0

        output_dict = {'id': [], 'raw_code': [], 'raw_comment': [], 'tokenized_comment': [], 'process_comment': [],
                       'feature': []}

        for line in tqdm(lines):
            data = json.loads(line.strip())
            if data['partition'] == file and data['language'] == 'java':
                code_id, raw_code, raw_comment = data['sha'], data['code'].strip(), data['docstring'].strip()
                code, comment = ' '.join(data['code_tokens']), ' '.join(data['docstring_tokens'])
                comment_v2 = ''.join(data['docstring_tokens'])
                feature_list = []
                output_dict['id'].append(code_id)
                output_dict['raw_code'].append(raw_code)
                output_dict['raw_comment'].append(raw_comment)
                output_dict['tokenized_comment'].append(comment)
                clean_code = formatFilter.clean_code_format(raw_code, 'java')
                process_code = formatFilter.cleanCommentedCode(raw_code)
                process_comment = formatFilter.splitFirstSentence(
                    BeautifulSoup(raw_comment, "html.parser").get_text()).strip()
                output_dict['process_comment'].append(process_comment)

                if clean_code == '':
                    all_commented_code += 1
                    feature_list.append('all_commented_code')
                else:
                    if code_dict.get(clean_code) is None:
                        code_dict[clean_code] = [[code_id], 1]
                    else:
                        code_dict[clean_code][0].append(code_id)
                        code_dict[clean_code][1] += 1

                        if formatFilter.ifALLCommentedOut(raw_code):
                            feature_list.append('all_commented_code')
                            all_commented_code += 1
                if formatFilter.findDeadCode(process_code, 'java'):
                    feature_list.append('DeadCode')
                    print(process_code, raw_code, raw_comment)
                    deadCode_num += 1

                # if not formatFilter.ifALLCommentedOut(raw_code):
                #     if formatFilter.findExistComment(raw_code):
                #         feature_list.append('partial comment')
                #         partialCommented_num += 1

                if formatFilter.findCommonFunc(process_code, 'java'):
                    feature_list.append('CommonFunc')
                    commonFunc_num += 1
                # else:
                #     if formatFilter.findAutoGenerated(process_comment, raw_comment):
                #         feature_list.append('CommonFunc')
                #         commonFunc_num += 1

                if not formatFilter.isEnglish(comment_v2):
                    nonEnglish_num += 1
                    feature_list.append('nonEnglish')
                else:
                    if formatFilter.findPunctuation(comment_v2):
                        nonEnglish_num += 1
                        feature_list.append('nonEnglish')

                if formatFilter.findInterrogation(comment.strip()):
                    feature_list.append('interrogation')
                    interrogation_num += 1
                if formatFilter.ifJavaTag(comment_v2):
                    feature_list.append('url')
                    url_num += 1
                else:
                    if formatFilter.findUrl(comment_v2):
                        feature_list.append('url')
                        url_num += 1
                    else:
                        if BeautifulSoup(comment_v2, "html.parser").get_text() != comment_v2:
                            feature_list.append('url')
                            url_num += 1

                firstfirst = formatFilter.findFirstSentence(comment, process_comment)
                if firstfirst == 'less':
                    feature_list.append('less')
                    firstSentence_num += 1
                    less_num += 1

                output_dict['feature'].append(feature_list)
            else:
                raise Exception('error')

        for id1, _ in zip(output_dict['id'], output_dict['feature']):
            if _:
                changedComment_num += 1
                if 'less' in _ or 'more' in _ or 'camelcaseSplit' in _ or 'html' in _ or \
                        'url' in _ or 'javaTag' in _ or 'interrogation' in _ or 'nonEnglish' in _:
                    comment_change_num += 1
                if 'all_commented_code' in _ or 'DeadCode' in _ or 'partial comment' in _ or 'CommonFunc' in _:
                    code_change_num += 1
                else:
                    unchanged_list_code.append(id1)
            else:
                unchanged_list.append(id1)
                unchanged_list_code.append(id1)
        print("changedComment_num", changedComment_num, 'ratio', round(changedComment_num / file_number * 100, 2))
        print("comment_change_num", comment_change_num, 'ratio', round(comment_change_num / file_number * 100, 2))
        print("code_change_num", code_change_num, 'ratio', round(code_change_num / file_number * 100, 2))
        print("firstSentence_num", firstSentence_num, 'ratio',
              round(firstSentence_num / file_number * 100, 2))
        print('remove_tag_num', javaTag_num, 'ratio', round(javaTag_num / file_number * 100, 2))
        print('remove_url_num', url_num, 'ratio', round(url_num / file_number * 100, 2))
        print('remove_nonEnglish_num', nonEnglish_num, 'ratio', round(nonEnglish_num / file_number * 100, 2))
        print('remove_interrogation_num', interrogation_num, 'ratio',
              round(interrogation_num / file_number * 100, 2))
        print('partial_comment_num', partialCommented_num, 'ratio', round(partialCommented_num / file_number * 100, 2))
        print('html_num', html_num, 'ratio', round(html_num / file_number * 100, 2))
        print('commonJavaFunc_num', commonFunc_num, 'ratio', round(commonFunc_num / file_number * 100, 2))
        print('punctuation_num', punctuation_num, 'ratio', round(punctuation_num / file_number * 100, 2))
        print('DeadCode_num', deadCode_num, 'ratio', round(deadCode_num / file_number * 100, 2))
        print('all_commented', all_commented_code, 'ratio', round(all_commented_code / file_number * 100, 2))
        print('more', more_num, 'ratio', round(more_num / file_number * 100, 2))
        print('less', less_num, 'ratio', round(less_num / file_number * 100, 2))
        # pd.DataFrame(output_dict).to_excel(f'./sample_data/codeSearchNet_{file}.xlsx',
        #                                    engine='xlsxwriter', index=False)  # , engine='xlsxwriter'
        print("f=========================================")

        total_num += file_number
        total_changedComment_num += changedComment_num
        total_comment_change_num += comment_change_num
        total_code_change_num += code_change_num
        total_firstSentence_num += firstSentence_num
        total_html_num += html_num
        total_javaTag_num += javaTag_num
        total_interrogation_num += interrogation_num
        total_url_num += url_num
        total_nonEnglish_num += nonEnglish_num
        total_partialCommented_num += partialCommented_num
        total_commonFunc_num += commonFunc_num
        total_punctuation_num += punctuation_num
        total_deadCode_num += deadCode_num
        total_all_commented_code += all_commented_code
        total_more_num += more_num
        total_less_num += less_num

    dup_num = 0
    for ids, number in code_dict.values():
        if number > 1:
            # print(ids, number)
            dup_num += number - 1

            temp_num = 0
            for id1 in ids:
                if id1 in unchanged_list:
                    temp_num += 1

            total_changedComment_num += max(temp_num - 1, 0)
            temp_code_num = 0
            for id1 in ids:
                if id1 in unchanged_list_code:
                    temp_code_num += 1
            total_code_change_num += max(temp_code_num - 1, 0)

    print(dup_num, total_num)
    print('duplicate_num', dup_num, 'ratio', round(dup_num / total_num * 100, 2))

    print("changedComment_num", total_changedComment_num, 'ratio', round(total_changedComment_num / total_num * 100, 2))
    print("total_comment_change_num", total_comment_change_num, 'ratio',
          round(total_comment_change_num / total_num * 100, 2))
    print("total_code_change_num", total_code_change_num, 'ratio', round(total_code_change_num / total_num * 100, 2))
    print("firstSentence_num", total_firstSentence_num, 'ratio',
          round(total_firstSentence_num / total_num * 100, 2))
    print('remove_tag_num', total_javaTag_num, 'ratio', round(total_javaTag_num / total_num * 100, 2))
    print('remove_url_num', total_url_num, 'ratio', round(total_url_num / total_num * 100, 2))
    print('remove_nonEnglish_num', total_nonEnglish_num, 'ratio', round(total_nonEnglish_num / total_num * 100, 2))
    print('remove_interrogation_num', total_interrogation_num, 'ratio',
          round(total_interrogation_num / total_num * 100, 2))
    print('commentedOut_num', total_partialCommented_num, 'ratio',
          round(total_partialCommented_num / total_num * 100, 2))
    print('html_num', total_html_num, 'ratio', round(total_html_num / total_num * 100, 2))
    print('commonJavaFunc_num', total_commonFunc_num, 'ratio', round(total_commonFunc_num / total_num * 100, 2))
    print('punctuation_num', total_punctuation_num, 'ratio', round(total_punctuation_num / total_num * 100, 2))
    print('DeadCode_num', total_deadCode_num, 'ratio', round(total_deadCode_num / total_num * 100, 2))
    print('all_commented', total_all_commented_code, 'ratio', round(total_all_commented_code / total_num * 100, 2))
    print('more', total_more_num, 'ratio', round(total_more_num / total_num * 100, 2))
    print('less', total_less_num, 'ratio', round(total_less_num / total_num * 100, 2))

# changedComment_num 322811 ratio 64.99
# firstSentence_num 163714 ratio 32.96
# camelcase_split_num 0 ratio 0.0
# remove_autogenerated_code_num 8747 ratio 1.76
# remove_tag_num 67469 ratio 13.58
# remove_rawCommentError_num 0 ratio 0.0
# remove_url_num 1446 ratio 0.29
# remove_nonEnglish_num 15190 ratio 3.06
# remove_uninformative_num 10649 ratio 2.14
# remove_interrogation_num 3433 ratio 0.69
# remove_nonFunc_num 4043 ratio 0.81
# commentedOut_num 109133 ratio 21.97
# html_num 62552 ratio 12.59
# commonJavaFunc_num 65119 ratio 13.11
# punctuation_num 30052 ratio 6.05


# duplicate_num 0 ratio 0.0
# changedComment_num 165284 ratio 33.28
# firstSentence_num 0 ratio 0.0
# remove_tag_num 52327 ratio 10.54
# remove_url_num 4962 ratio 1.0
# remove_nonEnglish_num 15379 ratio 3.1
# remove_interrogation_num 3535 ratio 0.71
# commentedOut_num 0 ratio 0.0
# html_num 83584 ratio 16.83
# commonJavaFunc_num 7865 ratio 1.58
# punctuation_num 30052 ratio 6.05
# DeadCode_num 117 ratio 0.02

import json
import pandas as pd
import re
import string
from re import finditer
from bs4 import BeautifulSoup
from tqdm import tqdm


class cleanFormat(object):

    def splitFirstSentence(self, text):
        clean_pattern1 = re.compile('([\s/*=-]+)$|^([\s/*=-]+)')
        clean_pattern2 = re.compile('^([\s*]+)')

        def func(t):
            t = t.strip().replace('&nbsp;', ' ')
            return re.sub(clean_pattern2, '', t).strip()

        text_list = [func(t) for t in re.sub(clean_pattern1, '', text).split('\n')]

        pattern1 = re.compile(r'([^\n]+?)(((?<!i\.e)(?<!e\.g)(?<!\beg)(?<!\bie))\.\s|\.$)')
        pattern2 = re.compile(r'^[A-Z@]|(\(?i\.e\.)|(\(?e\.g\.)|(\(?eg\.)|(\(?ie\.)[\s\S]+')

        start = 0
        for i in range(len(text_list)):
            if text_list[i] == '':
                start = i + 1
            else:
                break

        comment = ''
        for idx, text in enumerate(text_list[start:]):
            if comment == '':
                obj = re.match(pattern1, text)
                if obj:
                    return obj[0]
                else:
                    comment += text
            else:
                if text == '' or re.match(pattern2, text) is not None:
                    return comment
                else:
                    obj = re.match(pattern1, text)
                    if obj:
                        return comment + ' ' + obj[0]
                    else:
                        comment = comment + ' ' + text

        return comment.strip()

    def isEnglish(self, text):
        return text.isascii()

    def findExistComment(self, rawCode):
        rawCode = rawCode.strip()
        p0 = re.compile('^(//|/\*)')
        p1 = re.compile(';\n*(//|/\*)')
        text_list = [t.strip() for t in rawCode.split('\n')]
        for row in text_list:
            if row != '' and (p0.search(row) or p1.search(row)):
                return True
        return False

    def ifALLCommentedOut(self, rawCode):
        rawCode = rawCode.strip()
        p0 = re.compile('(^/\*)')
        if p0.search(rawCode):
            # print(rawCode)
            return True
        p1 = re.compile('(^//)')
        text_list = [t.strip() for t in rawCode.split('\n')]
        for row in text_list:
            if row != '' and not p1.search(row):
                return False
        return True

    def cleanCommentedCode(self, rawCode):
        p = re.compile('(^//)')
        text_list = [t.strip() for t in rawCode.strip().split('\n')]
        new_list = []
        for row in text_list:
            if row != '' and not p.search(row):
                new_list.append(row)
        return '\n'.join(new_list)

    def ifJavaTag(self, text):
        javaTag_pattern1 = re.compile('\{@')  # re.compile('\{@[^\s]+\s[^}]+\}')
        javaTag_pattern2 = re.compile(
            '@(see|throws|link|code|inheritdoc|return|param|author|version|exception|deprecated)')
        if javaTag_pattern1.search(text) or javaTag_pattern2.search(text):
            return True
        else:
            return False

    def findCommonFunc(self, rawCode, language):
        text_list = [t.strip() for t in rawCode.strip().split('\n')]
        text = text_list[0]
        pattern1 = re.compile(' toString\(')
        pattern2 = re.compile('([ _]test[A-Z_(])|([a-z_ ]Test[A-Z_(])')
        pattern3 = re.compile(' [a-zA-Z_]*Constructor[s]?\(')
        pattern5 = re.compile(' (set|get)[A-Z][a-zA-Z_]+\(')

        pattern6 = re.compile('__str__')
        if language == 'java':
            if pattern1.search(text) or pattern2.search(text) or pattern3.search(text):
                return True
            elif pattern5.search(text) and len(text_list) <= 3:
                return True
            else:
                return False
        elif language == 'python':
            if pattern2.search(text) or pattern6.search(text):
                return True
            else:
                return False
        else:
            return False

    def findDeadCode(self, rawCode, language):

        if language == 'java':
            pattern = re.compile('(?!\{\s+\})\{[\s\S]+\}')
            if pattern.search(rawCode):
                return False
            else:
                return True
        else:
            count = 0
            text_list = [t.strip() for t in rawCode.strip().split('\n')]
            for row in text_list:
                if row != '':
                    count += 1
            if count >= 3:
                return False
            else:
                return True

    def shortComment(self, text):
        if len(text.strip().split()) < 3:
            return True
        else:
            return False

    def findPunctuation(self, text):
        p = re.compile('[a-zA-Z]')
        if p.search(text):
            return False
        else:
            return True

    def findInterrogation(self, text):
        url_pattern = re.compile(
            '(http|ftp|https):\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])')
        text = re.sub(url_pattern, '', text)
        pattern1 = re.compile(r'(?i)^(why\b|how\b|what[\']?[s]?\b|where\b|is\b|are\b)')
        pattern2 = re.compile(r'(?i)(\bhow to\b|\bwhy\b|\bwhat (is|was|are)\b)')
        if '?' in text:
            return True
        elif pattern1.search(text) or pattern2.search(text):
            return True
        else:
            return False

    def findUrl(self, text):
        html_pattern = re.compile(
            '(http|ftp|https):\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])')
        if html_pattern.search(text):
            return True
        else:
            return False

    def clean_code_format(self, raw_code, language):
        if language == 'java':
            import javalang
            try:
                tokens = list(javalang.tokenizer.tokenize(raw_code))
                tks_list = []
                for tk in tokens:
                    tks_list.append(tk.value)

                code = " ".join(tks_list)
                return code
            except javalang.tokenizer.LexerError:
                return ''

    def findFirstSentence(self, tokenized_comment, process_comment):
        def cleanPunc(text):
            text = text.translate(str.maketrans(string.punctuation, ' ' * len(string.punctuation)))
            text = re.sub(r'\s+', ' ', text)
            return text.lower().strip()

        rawraw = cleanPunc(BeautifulSoup(tokenized_comment, "html.parser").get_text())
        processprocess = cleanPunc(BeautifulSoup(process_comment, "html.parser").get_text())
        if rawraw == processprocess:
            return False
        else:
            if len(rawraw.split()) > len(processprocess.split()):
                return 'more'
            else:
                return 'less'

    def findAutoGenerated(self, text, raw_text=None):
        auto_generated_pattern1 = re.compile(r'(?i)@[a-zA-Z]*generated\b')
        auto_generated_pattern2 = re.compile('(?i)^([aA]uto[-\s]generated)')
        auto_generated_pattern3 = re.compile('(?i)^(This method initializes)')
        auto_generated_pattern4 = re.compile('(?i)^(This method was generated by)')
        if raw_text is not None:
            if auto_generated_pattern1.search(raw_text):
                return True

        if auto_generated_pattern2.search(text) or auto_generated_pattern3.search(
                text) or auto_generated_pattern4.search(text):
            return True
        else:
            return False


if __name__ == '__main__':
    base_path = './../../../../Downloads/code2comment_dataset/dataset/CodeSearchNet_java_dataset/CodeSearchNet/java/'

    total_num = 0
    total_changedComment_num = 0
    total_code_change_num = 0
    total_comment_change_num = 0
    total_deadCode_num, total_commonFunc_num, total_nonEnglish_num, total_firstSentence_num, total_html_num, total_javaTag_num, total_interrogation_num = 0, 0, 0, 0, 0, 0, 0
    total_partialCommented_num, total_url_num, total_punctuation_num, total_all_commented_code, total_more_num, total_less_num = 0, 0, 0, 0, 0, 0

    code_dict = {}
    unchanged_list = []
    unchanged_list_code = []
    formatFilter = cleanFormat()
    for file in ['train', 'valid', 'test']:
        path = base_path + f'{file}/java_{file}_0.jsonl'
        with open(path, 'r') as f:
            lines = f.readlines()

        if file == 'train':
            for idx in range(1, 16):
                with open(base_path + f'{file}/java_{file}_{idx}.jsonl', 'r') as f:
                    lines += f.readlines()

        file_number = len(lines)
        changedComment_num = 0
        code_change_num = 0
        comment_change_num = 0
        deadCode_num, commonFunc_num, nonEnglish_num, firstSentence_num, html_num, javaTag_num, interrogation_num = 0, 0, 0, 0, 0, 0, 0
        partialCommented_num, url_num, punctuation_num, all_commented_code, more_num, less_num = 0, 0, 0, 0, 0, 0

        output_dict = {'id': [], 'raw_code': [], 'raw_comment': [], 'tokenized_comment': [], 'process_comment': [],
                       'feature': []}

        for line in tqdm(lines):
            data = json.loads(line.strip())
            if data['partition'] == file and data['language'] == 'java':
                code_id, raw_code, raw_comment = data['sha'], data['code'].strip(), data['docstring'].strip()
                code, comment = ' '.join(data['code_tokens']), ' '.join(data['docstring_tokens'])
                comment_v2 = ''.join(data['docstring_tokens'])
                feature_list = []
                output_dict['id'].append(code_id)
                output_dict['raw_code'].append(raw_code)
                output_dict['raw_comment'].append(raw_comment)
                output_dict['tokenized_comment'].append(comment)
                clean_code = formatFilter.clean_code_format(raw_code, 'java')
                process_code = formatFilter.cleanCommentedCode(raw_code)
                process_comment = formatFilter.splitFirstSentence(
                    BeautifulSoup(raw_comment, "html.parser").get_text()).strip()
                output_dict['process_comment'].append(process_comment)

                if clean_code == '':
                    all_commented_code += 1
                    feature_list.append('all_commented_code')
                else:
                    if code_dict.get(clean_code) is None:
                        code_dict[clean_code] = [[code_id], 1]
                    else:
                        code_dict[clean_code][0].append(code_id)
                        code_dict[clean_code][1] += 1

                        if formatFilter.ifALLCommentedOut(raw_code):
                            feature_list.append('all_commented_code')
                            all_commented_code += 1
                if formatFilter.findDeadCode(process_code, 'java'):
                    feature_list.append('DeadCode')
                    print(process_code, raw_code, raw_comment)
                    deadCode_num += 1

                # if not formatFilter.ifALLCommentedOut(raw_code):
                #     if formatFilter.findExistComment(raw_code):
                #         feature_list.append('partial comment')
                #         partialCommented_num += 1

                if formatFilter.findCommonFunc(process_code, 'java'):
                    feature_list.append('CommonFunc')
                    commonFunc_num += 1
                # else:
                #     if formatFilter.findAutoGenerated(process_comment, raw_comment):
                #         feature_list.append('CommonFunc')
                #         commonFunc_num += 1

                if not formatFilter.isEnglish(comment_v2):
                    nonEnglish_num += 1
                    feature_list.append('nonEnglish')
                else:
                    if formatFilter.findPunctuation(comment_v2):
                        nonEnglish_num += 1
                        feature_list.append('nonEnglish')

                if formatFilter.findInterrogation(comment.strip()):
                    feature_list.append('interrogation')
                    interrogation_num += 1
                if formatFilter.ifJavaTag(comment_v2):
                    feature_list.append('url')
                    url_num += 1
                else:
                    if formatFilter.findUrl(comment_v2):
                        feature_list.append('url')
                        url_num += 1
                    else:
                        if BeautifulSoup(comment_v2, "html.parser").get_text() != comment_v2:
                            feature_list.append('url')
                            url_num += 1

                firstfirst = formatFilter.findFirstSentence(comment, process_comment)
                if firstfirst == 'less':
                    feature_list.append('less')
                    firstSentence_num += 1
                    less_num += 1

                output_dict['feature'].append(feature_list)
            else:
                raise Exception('error')

        for id1, _ in zip(output_dict['id'], output_dict['feature']):
            if _:
                changedComment_num += 1
                if 'less' in _ or 'more' in _ or 'camelcaseSplit' in _ or 'html' in _ or \
                        'url' in _ or 'javaTag' in _ or 'interrogation' in _ or 'nonEnglish' in _:
                    comment_change_num += 1
                if 'all_commented_code' in _ or 'DeadCode' in _ or 'partial comment' in _ or 'CommonFunc' in _:
                    code_change_num += 1
                else:
                    unchanged_list_code.append(id1)
            else:
                unchanged_list.append(id1)
                unchanged_list_code.append(id1)
        print("changedComment_num", changedComment_num, 'ratio', round(changedComment_num / file_number * 100, 2))
        print("comment_change_num", comment_change_num, 'ratio', round(comment_change_num / file_number * 100, 2))
        print("code_change_num", code_change_num, 'ratio', round(code_change_num / file_number * 100, 2))
        print("firstSentence_num", firstSentence_num, 'ratio',
              round(firstSentence_num / file_number * 100, 2))
        print('remove_tag_num', javaTag_num, 'ratio', round(javaTag_num / file_number * 100, 2))
        print('remove_url_num', url_num, 'ratio', round(url_num / file_number * 100, 2))
        print('remove_nonEnglish_num', nonEnglish_num, 'ratio', round(nonEnglish_num / file_number * 100, 2))
        print('remove_interrogation_num', interrogation_num, 'ratio',
              round(interrogation_num / file_number * 100, 2))
        print('partial_comment_num', partialCommented_num, 'ratio', round(partialCommented_num / file_number * 100, 2))
        print('html_num', html_num, 'ratio', round(html_num / file_number * 100, 2))
        print('commonJavaFunc_num', commonFunc_num, 'ratio', round(commonFunc_num / file_number * 100, 2))
        print('punctuation_num', punctuation_num, 'ratio', round(punctuation_num / file_number * 100, 2))
        print('DeadCode_num', deadCode_num, 'ratio', round(deadCode_num / file_number * 100, 2))
        print('all_commented', all_commented_code, 'ratio', round(all_commented_code / file_number * 100, 2))
        print('more', more_num, 'ratio', round(more_num / file_number * 100, 2))
        print('less', less_num, 'ratio', round(less_num / file_number * 100, 2))
        # pd.DataFrame(output_dict).to_excel(f'./sample_data/codeSearchNet_{file}.xlsx',
        #                                    engine='xlsxwriter', index=False)  # , engine='xlsxwriter'
        print("f=========================================")

        total_num += file_number
        total_changedComment_num += changedComment_num
        total_comment_change_num += comment_change_num
        total_code_change_num += code_change_num
        total_firstSentence_num += firstSentence_num
        total_html_num += html_num
        total_javaTag_num += javaTag_num
        total_interrogation_num += interrogation_num
        total_url_num += url_num
        total_nonEnglish_num += nonEnglish_num
        total_partialCommented_num += partialCommented_num
        total_commonFunc_num += commonFunc_num
        total_punctuation_num += punctuation_num
        total_deadCode_num += deadCode_num
        total_all_commented_code += all_commented_code
        total_more_num += more_num
        total_less_num += less_num

    dup_num = 0
    for ids, number in code_dict.values():
        if number > 1:
            # print(ids, number)
            dup_num += number - 1

            temp_num = 0
            for id1 in ids:
                if id1 in unchanged_list:
                    temp_num += 1

            total_changedComment_num += max(temp_num - 1, 0)
            temp_code_num = 0
            for id1 in ids:
                if id1 in unchanged_list_code:
                    temp_code_num += 1
            total_code_change_num += max(temp_code_num - 1, 0)

    print(dup_num, total_num)
    print('duplicate_num', dup_num, 'ratio', round(dup_num / total_num * 100, 2))

    print("changedComment_num", total_changedComment_num, 'ratio', round(total_changedComment_num / total_num * 100, 2))
    print("total_comment_change_num", total_comment_change_num, 'ratio',
          round(total_comment_change_num / total_num * 100, 2))
    print("total_code_change_num", total_code_change_num, 'ratio', round(total_code_change_num / total_num * 100, 2))
    print("firstSentence_num", total_firstSentence_num, 'ratio',
          round(total_firstSentence_num / total_num * 100, 2))
    print('remove_tag_num', total_javaTag_num, 'ratio', round(total_javaTag_num / total_num * 100, 2))
    print('remove_url_num', total_url_num, 'ratio', round(total_url_num / total_num * 100, 2))
    print('remove_nonEnglish_num', total_nonEnglish_num, 'ratio', round(total_nonEnglish_num / total_num * 100, 2))
    print('remove_interrogation_num', total_interrogation_num, 'ratio',
          round(total_interrogation_num / total_num * 100, 2))
    print('commentedOut_num', total_partialCommented_num, 'ratio',
          round(total_partialCommented_num / total_num * 100, 2))
    print('html_num', total_html_num, 'ratio', round(total_html_num / total_num * 100, 2))
    print('commonJavaFunc_num', total_commonFunc_num, 'ratio', round(total_commonFunc_num / total_num * 100, 2))
    print('punctuation_num', total_punctuation_num, 'ratio', round(total_punctuation_num / total_num * 100, 2))
    print('DeadCode_num', total_deadCode_num, 'ratio', round(total_deadCode_num / total_num * 100, 2))
    print('all_commented', total_all_commented_code, 'ratio', round(total_all_commented_code / total_num * 100, 2))
    print('more', total_more_num, 'ratio', round(total_more_num / total_num * 100, 2))
    print('less', total_less_num, 'ratio', round(total_less_num / total_num * 100, 2))

# changedComment_num 322811 ratio 64.99
# firstSentence_num 163714 ratio 32.96
# camelcase_split_num 0 ratio 0.0
# remove_autogenerated_code_num 8747 ratio 1.76
# remove_tag_num 67469 ratio 13.58
# remove_rawCommentError_num 0 ratio 0.0
# remove_url_num 1446 ratio 0.29
# remove_nonEnglish_num 15190 ratio 3.06
# remove_uninformative_num 10649 ratio 2.14
# remove_interrogation_num 3433 ratio 0.69
# remove_nonFunc_num 4043 ratio 0.81
# commentedOut_num 109133 ratio 21.97
# html_num 62552 ratio 12.59
# commonJavaFunc_num 65119 ratio 13.11
# punctuation_num 30052 ratio 6.05


# duplicate_num 0 ratio 0.0
# changedComment_num 165284 ratio 33.28
# firstSentence_num 0 ratio 0.0
# remove_tag_num 52327 ratio 10.54
# remove_url_num 4962 ratio 1.0
# remove_nonEnglish_num 15379 ratio 3.1
# remove_interrogation_num 3535 ratio 0.71
# commentedOut_num 0 ratio 0.0
# html_num 83584 ratio 16.83
# commonJavaFunc_num 7865 ratio 1.58
# punctuation_num 30052 ratio 6.05
# DeadCode_num 117 ratio 0.02

# print(rawCode) # re.compile('\{@[^\s]+\s[^}]+\}') #-]*[\w@?^=%&\/~+#-])') #-]*[\w@?^=%&\/~+#-])') # if not formatFilter.ifALLCommentedOut(raw_code): #     if formatFilter.findExistComment(raw_code): #         feature_list.append('partial comment') #         partialCommented_num += 1 # else: #     if formatFilter.findAutoGenerated(process_comment, raw_comment): #         feature_list.append('CommonFunc') #         commonFunc_num += 1 # pd.DataFrame(output_dict).to_excel(f'./sample_data/codeSearchNet_{file}.xlsx', #                                    engine='xlsxwriter', index=False)  # , engine='xlsxwriter' # print(ids, number) # changedComment_num 322811 ratio 64.99 # firstSentence_num 163714 ratio 32.96 # camelcase_split_num 0 ratio 0.0 # remove_autogenerated_code_num 8747 ratio 1.76 # remove_tag_num 67469 ratio 13.58 # remove_rawCommentError_num 0 ratio 0.0 # remove_url_num 1446 ratio 0.29 # remove_nonEnglish_num 15190 ratio 3.06 # remove_uninformative_num 10649 ratio 2.14 # remove_interrogation_num 3433 ratio 0.69 # remove_nonFunc_num 4043 ratio 0.81 # commentedOut_num 109133 ratio 21.97 # html_num 62552 ratio 12.59 # commonJavaFunc_num 65119 ratio 13.11 # punctuation_num 30052 ratio 6.05 # duplicate_num 0 ratio 0.0 # changedComment_num 165284 ratio 33.28 # firstSentence_num 0 ratio 0.0 # remove_tag_num 52327 ratio 10.54 # remove_url_num 4962 ratio 1.0 # remove_nonEnglish_num 15379 ratio 3.1 # remove_interrogation_num 3535 ratio 0.71 # commentedOut_num 0 ratio 0.0 # html_num 83584 ratio 16.83 # commonJavaFunc_num 7865 ratio 1.58 # punctuation_num 30052 ratio 6.05 # DeadCode_num 117 ratio 0.02