from nltk.tree import Tree

def left_branching(tree):
    t = Tree('NAN', [tree.leaves()[0]])
    for i, l in enumerate(tree.leaves()[1:]):
        t = Tree('NAN', [t, Tree('NAN', [l])])
    return t

def right_branching(tree):
    t = Tree('NAN', [tree.leaves()[-1]])
    for i, l in enumerate(reversed(tree.leaves()[:-1])):
        t = Tree('NAN', [Tree('NAN', [l]), t])
    return t

def random_branching(tree):
    import random
    t = [Tree('NAN', [i]) for i in tree.leaves()]
    while len(t) >= 2:
        idx = random.randrange(0,len(t)-1)
        t[idx:idx+2] = [Tree('NAN', [t[idx], t[idx+1]])]
    return t[0]

def remove_punctuation_from_tree(tree, ref):
    
    word_tags = set(['CC', 'CD', 'DT', 'EX', 'FW', 'IN', 'JJ', 'JJR', 'JJS', 'LS', 'MD', 'NN',
               'NNS', 'NNP', 'NNPS', 'PDT', 'POS', 'PRP', 'PRP$', 'RB', 'RBR', 'RBS',
               'RP', 'SYM', 'TO', 'UH', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ',
               'WDT', 'WP', 'WP$', 'WRB'])

    def recursive_remove_using_mask(tr, position, mask):
        if len(tr) == 1 and isinstance(tr[0], str):
            size = 1
            keep = mask[position]
            return size, keep

        size = 0
        keep = []
        for x in tr:
            xsize, xkeep = recursive_remove_using_mask(x, position + size, mask)
            size += xsize
            keep.append(xkeep)

        for i, xkeep in list(enumerate(keep))[::-1]:
            if not xkeep:
                del tr[i]

        keep = any(keep)
        return size, keep

    tokens = tree.leaves()
    part_of_speech = [x[1] for x in ref.pos()]
    mask = [x in word_tags for x in part_of_speech] # Tokens that are punctuation are given False in mask.
    new_tokens = [x for x, m in zip(tokens, mask) if m]
    assert len(new_tokens) > 0

    recursive_remove_using_mask(tree, 0, mask)
    assert len(tree.leaves()) == len(new_tokens), (tree.leaves(), new_tokens, tokens, mask)
    assert tuple(tree.leaves()) == tuple(new_tokens)

    return tree

def get_ptb_format_from_nltk_tree(tr):
    def helper(tr):
        if len(tr) == 1 and isinstance(tr[0], str):
            return f'({tr.label()} {tr[0]})'

        nodes = [helper(x) for x in tr]

        return f'({tr.label()} {" ".join(nodes)})'

    out = helper(tr)

    if out.startswith('(DT'):
        out = f'(S {out})'

    return out

def main(args):
    f_gold = open(args.ptb_path)
    f_pred = open(args.out_path, "w")
    for line in f_gold:
        tree = Tree.fromstring(line[:-1])
        #tree.pretty_print()
        if args.type == 'left':
            new_tree = left_branching(tree)
        elif args.type == 'right':
            new_tree = right_branching(tree)
        elif args.type == 'random':
            new_tree = random_branching(tree)
        print(" ".join(str(new_tree).split()), file=f_pred)
        #new_tree.pretty_print()
        #break
    f_gold.close()
    f_pred.close()

def evalb(args):
    import os

    with open(args.ptb_path) as f_gold:
        golds = []
        for line in f_gold:
            golds.append(Tree.fromstring(line))

    with open(args.out_path) as f_pred:
        preds = []
        for line in f_pred:
            preds.append(Tree.fromstring(line))
    
    assert len(golds) == len(preds), f"The gold and pred files must have same number of sentences. {len(preds)} != {len(golds)}"

    preds = [remove_punctuation_from_tree(pred, ref=gold) for gold, pred in zip(golds, preds)]
    golds = [remove_punctuation_from_tree(gold, ref=gold) for gold, pred in zip(golds, preds)]

    assert all(len(x.leaves()) > 0 for x in preds)
    assert all(len(x.leaves()) > 0 for x in golds)
    #assert all(len(p.leaves()) == len(g.leaves()) for g, p in zip(gold, pred))
    for g, p in zip(golds, preds):
        if len(p.leaves()) != len(g.leaves()):
            print(p.leaves())
            print(g.leaves())
            raise Exception

    # Serialize as strings.
    golds = [get_ptb_format_from_nltk_tree(g) for g, p in zip(golds, preds)]
    preds = [get_ptb_format_from_nltk_tree(p) for g, p in zip(golds, preds)]

    # Write new intermediate files.
    new_pred_file = os.path.join(args.out, 'pred.txt')
    new_gold_file = os.path.join(args.out, 'gold.txt')

    with open(new_pred_file, 'w') as f:
        for parse in preds:
            f.write(parse + '\n')

    with open(new_gold_file, 'w') as f:
        for parse in golds:
            f.write(parse + '\n')

    # Run EVALB.
    evalb_exe = os.path.join(args.evalb, 'evalb')
    evalb_out_file = os.path.join(args.out, 'evalb.out')

    assert os.path.exists(evalb_exe), "Did not detect evalb executable. Try `(cd EVALB && make)`."
    assert os.path.exists(args.evalb_config)

    evalb_command = '{evalb} -p {evalb_config} {gold} {pred} > {out}'.format(
        evalb=os.path.join(args.evalb, 'evalb'),
        evalb_config=args.evalb_config,
        gold=new_gold_file,
        pred=new_pred_file,
        out=evalb_out_file)

    print(f'\nRunning: {evalb_command}')
    os.system(evalb_command)

    print(f'\nResults are ready at: {evalb_out_file}')

    print(f'\n==== PREVIEW OF RESULTS ({evalb_out_file}) ====\n')
    os.system(f'tail -n 27 {evalb_out_file}')
    print('')

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("--ptb_path", type=str)
    parser.add_argument("--out_path", type=str)
    parser.add_argument('--type', choices=('left', 'right', 'random'))
    
    parser.add_argument('--out', type=str, required=True, help='Directory to write intermediates files for EVALB and results.')
    parser.add_argument('--evalb', type=str, required=True, help='Path to EVALB directory.')
    parser.add_argument('--evalb_config', type=str, required=True, help='Path to EVALB configuration file.')
    parser.add_argument('--max_length', default=None, type=str, help='Max length after removing punctuation.')
    args = parser.parse_args()

    main(args)
    evalb(args)
from nltk.tree import Tree

def left_branching(tree):
    t = Tree('NAN', [tree.leaves()[0]])
    for i, l in enumerate(tree.leaves()[1:]):
        t = Tree('NAN', [t, Tree('NAN', [l])])
    return t

def right_branching(tree):
    t = Tree('NAN', [tree.leaves()[-1]])
    for i, l in enumerate(reversed(tree.leaves()[:-1])):
        t = Tree('NAN', [Tree('NAN', [l]), t])
    return t

def random_branching(tree):
    import random
    t = [Tree('NAN', [i]) for i in tree.leaves()]
    while len(t) >= 2:
        idx = random.randrange(0,len(t)-1)
        t[idx:idx+2] = [Tree('NAN', [t[idx], t[idx+1]])]
    return t[0]

def remove_punctuation_from_tree(tree, ref):
    
    word_tags = set(['CC', 'CD', 'DT', 'EX', 'FW', 'IN', 'JJ', 'JJR', 'JJS', 'LS', 'MD', 'NN',
               'NNS', 'NNP', 'NNPS', 'PDT', 'POS', 'PRP', 'PRP$', 'RB', 'RBR', 'RBS',
               'RP', 'SYM', 'TO', 'UH', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ',
               'WDT', 'WP', 'WP$', 'WRB'])

    def recursive_remove_using_mask(tr, position, mask):
        if len(tr) == 1 and isinstance(tr[0], str):
            size = 1
            keep = mask[position]
            return size, keep

        size = 0
        keep = []
        for x in tr:
            xsize, xkeep = recursive_remove_using_mask(x, position + size, mask)
            size += xsize
            keep.append(xkeep)

        for i, xkeep in list(enumerate(keep))[::-1]:
            if not xkeep:
                del tr[i]

        keep = any(keep)
        return size, keep

    tokens = tree.leaves()
    part_of_speech = [x[1] for x in ref.pos()]
    mask = [x in word_tags for x in part_of_speech] # Tokens that are punctuation are given False in mask.
    new_tokens = [x for x, m in zip(tokens, mask) if m]
    assert len(new_tokens) > 0

    recursive_remove_using_mask(tree, 0, mask)
    assert len(tree.leaves()) == len(new_tokens), (tree.leaves(), new_tokens, tokens, mask)
    assert tuple(tree.leaves()) == tuple(new_tokens)

    return tree

def get_ptb_format_from_nltk_tree(tr):
    def helper(tr):
        if len(tr) == 1 and isinstance(tr[0], str):
            return f'({tr.label()} {tr[0]})'

        nodes = [helper(x) for x in tr]

        return f'({tr.label()} {" ".join(nodes)})'

    out = helper(tr)

    if out.startswith('(DT'):
        out = f'(S {out})'

    return out

def main(args):
    f_gold = open(args.ptb_path)
    f_pred = open(args.out_path, "w")
    for line in f_gold:
        tree = Tree.fromstring(line[:-1])
        #tree.pretty_print()
        if args.type == 'left':
            new_tree = left_branching(tree)
        elif args.type == 'right':
            new_tree = right_branching(tree)
        elif args.type == 'random':
            new_tree = random_branching(tree)
        print(" ".join(str(new_tree).split()), file=f_pred)
        #new_tree.pretty_print()
        #break
    f_gold.close()
    f_pred.close()

def evalb(args):
    import os

    with open(args.ptb_path) as f_gold:
        golds = []
        for line in f_gold:
            golds.append(Tree.fromstring(line))

    with open(args.out_path) as f_pred:
        preds = []
        for line in f_pred:
            preds.append(Tree.fromstring(line))
    
    assert len(golds) == len(preds), f"The gold and pred files must have same number of sentences. {len(preds)} != {len(golds)}"

    preds = [remove_punctuation_from_tree(pred, ref=gold) for gold, pred in zip(golds, preds)]
    golds = [remove_punctuation_from_tree(gold, ref=gold) for gold, pred in zip(golds, preds)]

    assert all(len(x.leaves()) > 0 for x in preds)
    assert all(len(x.leaves()) > 0 for x in golds)
    #assert all(len(p.leaves()) == len(g.leaves()) for g, p in zip(gold, pred))
    for g, p in zip(golds, preds):
        if len(p.leaves()) != len(g.leaves()):
            print(p.leaves())
            print(g.leaves())
            raise Exception

    # Serialize as strings.
    golds = [get_ptb_format_from_nltk_tree(g) for g, p in zip(golds, preds)]
    preds = [get_ptb_format_from_nltk_tree(p) for g, p in zip(golds, preds)]

    # Write new intermediate files.
    new_pred_file = os.path.join(args.out, 'pred.txt')
    new_gold_file = os.path.join(args.out, 'gold.txt')

    with open(new_pred_file, 'w') as f:
        for parse in preds:
            f.write(parse + '\n')

    with open(new_gold_file, 'w') as f:
        for parse in golds:
            f.write(parse + '\n')

    # Run EVALB.
    evalb_exe = os.path.join(args.evalb, 'evalb')
    evalb_out_file = os.path.join(args.out, 'evalb.out')

    assert os.path.exists(evalb_exe), "Did not detect evalb executable. Try `(cd EVALB && make)`."
    assert os.path.exists(args.evalb_config)

    evalb_command = '{evalb} -p {evalb_config} {gold} {pred} > {out}'.format(
        evalb=os.path.join(args.evalb, 'evalb'),
        evalb_config=args.evalb_config,
        gold=new_gold_file,
        pred=new_pred_file,
        out=evalb_out_file)

    print(f'\nRunning: {evalb_command}')
    os.system(evalb_command)

    print(f'\nResults are ready at: {evalb_out_file}')

    print(f'\n==== PREVIEW OF RESULTS ({evalb_out_file}) ====\n')
    os.system(f'tail -n 27 {evalb_out_file}')
    print('')

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("--ptb_path", type=str)
    parser.add_argument("--out_path", type=str)
    parser.add_argument('--type', choices=('left', 'right', 'random'))
    
    parser.add_argument('--out', type=str, required=True, help='Directory to write intermediates files for EVALB and results.')
    parser.add_argument('--evalb', type=str, required=True, help='Path to EVALB directory.')
    parser.add_argument('--evalb_config', type=str, required=True, help='Path to EVALB configuration file.')
    parser.add_argument('--max_length', default=None, type=str, help='Max length after removing punctuation.')
    args = parser.parse_args()

    main(args)
    evalb(args)
# Tokens that are punctuation are given False in mask. #tree.pretty_print() #new_tree.pretty_print() #break #assert all(len(p.leaves()) == len(g.leaves()) for g, p in zip(gold, pred)) # Serialize as strings. # Write new intermediate files. # Run EVALB.