<reponame>xksteven/ethics_amti<gh_stars>0
import numpy as np
import pandas as pd

v = 2

df = pd.read_csv("saved_data{}.tsv".format(v), sep="\t", header=None)
print(df.shape)

def levenshteinDistance(s1, s2):
    if len(s1) > len(s2):
        s1, s2 = s2, s1

    distances = range(len(s1) + 1)
    for i2, c2 in enumerate(s2):
        distances_ = [i2+1]
        for i1, c1 in enumerate(s1):
            if c1 == c2:
                distances_.append(distances[i1])
            else:
                distances_.append(1 + min((distances[i1], distances[i1 + 1], distances_[-1])))
        distances = distances_
    return distances[-1]

# n = df.shape[0] // 2
# require_good_pair = True

good_sentences = []
bad_sentences = []

# TODO: make sure things work even when not exactly two...
# Use edit distance.

# gather a list of lists
# loop through. keep track of previous sentence. check if current sentence has small edit distance.
# if so, add it to the most recent list. Otherwise, append the most recent list.
sets = []
set = []
prev = ""
for i in range(df.shape[0]):
    s = df.iloc[i, 0]
    if levenshteinDistance(s, prev) < 8:
        set.append(i)
        if i == df.shape[0] - 1:
            sets.append(set)
    else:
        sets.append(set)
        set = [i]

    prev = s

sets = sets[1:]  # remove empty set
size = 0
for set in sets:
    size += len(set) / len(sets)

keep_sentences = []
dont_keep_sentences = []
for set in sets:
    if len(set) < 2:
        dont_keep_sentences.append(df.iloc[i, 0])
        continue
    for j in range(10):
        sentences = []
        to_compare_sentences = []
        for i in set:
            sentences.append(df.iloc[i, j])
            to_compare_sentences.append(df.iloc[i, j].lower().replace(" ", "").replace(".", ""))

        # loop through pairs of sentences; see if at least two agree.
        keep = ""
        for k1 in range(1, len(set)):
            for k2 in range(k1):
                t1, t2 = to_compare_sentences[k1], to_compare_sentences[k2]
                if t1 == t2:
                    keep = sentences[k1]
        if keep:
            keep_sentences.append(keep)
        else:
            dont_keep_sentences.append(sentences[0])

print(len(dont_keep_sentences))
print(len(keep_sentences))
df_cleaned = pd.DataFrame({"sentences": keep_sentences})
df_cleaned.to_csv("cleaned_results{}.tsv".format(v), sep="\t", header=None)

# df_cleaned_good = pd.DataFrame({"good": good_sentences})
# df_cleaned_wrong = pd.DataFrame({"wrong": bad_sentences})
# df_cleaned_good.to_csv("cleaned_good_results4.tsv", sep="\t", header=None)
# df_cleaned_wrong.to_csv("cleaned_wrong_results4.tsv", sep="\t", header=None)

# now for each set, see if there is a pair that agrees. if so, write to file along with the index.

# for i in range(n):
#     for j in range(5):
#         # good/not wrong sentence
#         s1, s2 = df.iloc[2*i, 2*j], df.iloc[2*i+1, 2*j]
#         s1_l, s2_l = s1.lower().replace(" ", "").replace(".", ""), s2.lower().replace(" ", "").replace(".", "")
#
#         # bad/wrong sentence
#         t1, t2 = df.iloc[2*i, 2*j+1], df.iloc[2*i+1, 2*j+1]
#         t1_l, t2_l = t1.lower().replace(" ", "").replace(".", ""), t2.lower().replace(" ", "").replace(".", "")
#
#
#         if require_good_pair:
#             if s1_l == s2_l and t1_l == t2_l:
#                 # save to file
#                 good_sentences.append(s1)
#                 bad_sentences.append(t1)
#             else:
#                 if s1_l != s2_l:
#                     print(s1, s2)
#                 else:
#                     print(t1, t2)
#         else:
#             if s1_l == s2_l:
#                 # save to file
#                 good_sentences.append(s1)
#             if t1_l == t2_l:
#                 bad_sentences.append(t1)
#
#
# if require_good_pair:
#     df_cleaned = pd.DataFrame({"not_wrong": good_sentences, "wrong": bad_sentences})
#     df_cleaned.to_csv("cleaned_results4.tsv", sep="\t", header=None)
# else:
#     df_cleaned_good = pd.DataFrame({"good": good_sentences})
#     df_cleaned_wrong = pd.DataFrame({"wrong": bad_sentences})
#     df_cleaned_good.to_csv("cleaned_good_results4.tsv", sep="\t", header=None)
#     df_cleaned_wrong.to_csv("cleaned_wrong_results4.tsv", sep="\t", header=None)

import numpy as np
import pandas as pd

v = 2

df = pd.read_csv("saved_data{}.tsv".format(v), sep="\t", header=None)
print(df.shape)

def levenshteinDistance(s1, s2):
    if len(s1) > len(s2):
        s1, s2 = s2, s1

    distances = range(len(s1) + 1)
    for i2, c2 in enumerate(s2):
        distances_ = [i2+1]
        for i1, c1 in enumerate(s1):
            if c1 == c2:
                distances_.append(distances[i1])
            else:
                distances_.append(1 + min((distances[i1], distances[i1 + 1], distances_[-1])))
        distances = distances_
    return distances[-1]

# n = df.shape[0] // 2
# require_good_pair = True

good_sentences = []
bad_sentences = []

# TODO: make sure things work even when not exactly two...
# Use edit distance.

# gather a list of lists
# loop through. keep track of previous sentence. check if current sentence has small edit distance.
# if so, add it to the most recent list. Otherwise, append the most recent list.
sets = []
set = []
prev = ""
for i in range(df.shape[0]):
    s = df.iloc[i, 0]
    if levenshteinDistance(s, prev) < 8:
        set.append(i)
        if i == df.shape[0] - 1:
            sets.append(set)
    else:
        sets.append(set)
        set = [i]

    prev = s

sets = sets[1:]  # remove empty set
size = 0
for set in sets:
    size += len(set) / len(sets)

keep_sentences = []
dont_keep_sentences = []
for set in sets:
    if len(set) < 2:
        dont_keep_sentences.append(df.iloc[i, 0])
        continue
    for j in range(10):
        sentences = []
        to_compare_sentences = []
        for i in set:
            sentences.append(df.iloc[i, j])
            to_compare_sentences.append(df.iloc[i, j].lower().replace(" ", "").replace(".", ""))

        # loop through pairs of sentences; see if at least two agree.
        keep = ""
        for k1 in range(1, len(set)):
            for k2 in range(k1):
                t1, t2 = to_compare_sentences[k1], to_compare_sentences[k2]
                if t1 == t2:
                    keep = sentences[k1]
        if keep:
            keep_sentences.append(keep)
        else:
            dont_keep_sentences.append(sentences[0])

print(len(dont_keep_sentences))
print(len(keep_sentences))
df_cleaned = pd.DataFrame({"sentences": keep_sentences})
df_cleaned.to_csv("cleaned_results{}.tsv".format(v), sep="\t", header=None)

# df_cleaned_good = pd.DataFrame({"good": good_sentences})
# df_cleaned_wrong = pd.DataFrame({"wrong": bad_sentences})
# df_cleaned_good.to_csv("cleaned_good_results4.tsv", sep="\t", header=None)
# df_cleaned_wrong.to_csv("cleaned_wrong_results4.tsv", sep="\t", header=None)

# now for each set, see if there is a pair that agrees. if so, write to file along with the index.

# for i in range(n):
#     for j in range(5):
#         # good/not wrong sentence
#         s1, s2 = df.iloc[2*i, 2*j], df.iloc[2*i+1, 2*j]
#         s1_l, s2_l = s1.lower().replace(" ", "").replace(".", ""), s2.lower().replace(" ", "").replace(".", "")
#
#         # bad/wrong sentence
#         t1, t2 = df.iloc[2*i, 2*j+1], df.iloc[2*i+1, 2*j+1]
#         t1_l, t2_l = t1.lower().replace(" ", "").replace(".", ""), t2.lower().replace(" ", "").replace(".", "")
#
#
#         if require_good_pair:
#             if s1_l == s2_l and t1_l == t2_l:
#                 # save to file
#                 good_sentences.append(s1)
#                 bad_sentences.append(t1)
#             else:
#                 if s1_l != s2_l:
#                     print(s1, s2)
#                 else:
#                     print(t1, t2)
#         else:
#             if s1_l == s2_l:
#                 # save to file
#                 good_sentences.append(s1)
#             if t1_l == t2_l:
#                 bad_sentences.append(t1)
#
#
# if require_good_pair:
#     df_cleaned = pd.DataFrame({"not_wrong": good_sentences, "wrong": bad_sentences})
#     df_cleaned.to_csv("cleaned_results4.tsv", sep="\t", header=None)
# else:
#     df_cleaned_good = pd.DataFrame({"good": good_sentences})
#     df_cleaned_wrong = pd.DataFrame({"wrong": bad_sentences})
#     df_cleaned_good.to_csv("cleaned_good_results4.tsv", sep="\t", header=None)
#     df_cleaned_wrong.to_csv("cleaned_wrong_results4.tsv", sep="\t", header=None)
# n = df.shape[0] // 2 # require_good_pair = True # TODO: make sure things work even when not exactly two... # Use edit distance. # gather a list of lists # loop through. keep track of previous sentence. check if current sentence has small edit distance. # if so, add it to the most recent list. Otherwise, append the most recent list. # remove empty set # loop through pairs of sentences; see if at least two agree. # df_cleaned_good = pd.DataFrame({"good": good_sentences}) # df_cleaned_wrong = pd.DataFrame({"wrong": bad_sentences}) # df_cleaned_good.to_csv("cleaned_good_results4.tsv", sep="\t", header=None) # df_cleaned_wrong.to_csv("cleaned_wrong_results4.tsv", sep="\t", header=None) # now for each set, see if there is a pair that agrees. if so, write to file along with the index. # for i in range(n): #     for j in range(5): #         # good/not wrong sentence #         s1, s2 = df.iloc[2*i, 2*j], df.iloc[2*i+1, 2*j] #         s1_l, s2_l = s1.lower().replace(" ", "").replace(".", ""), s2.lower().replace(" ", "").replace(".", "") # #         # bad/wrong sentence #         t1, t2 = df.iloc[2*i, 2*j+1], df.iloc[2*i+1, 2*j+1] #         t1_l, t2_l = t1.lower().replace(" ", "").replace(".", ""), t2.lower().replace(" ", "").replace(".", "") # # #         if require_good_pair: #             if s1_l == s2_l and t1_l == t2_l: #                 # save to file #                 good_sentences.append(s1) #                 bad_sentences.append(t1) #             else: #                 if s1_l != s2_l: #                     print(s1, s2) #                 else: #                     print(t1, t2) #         else: #             if s1_l == s2_l: #                 # save to file #                 good_sentences.append(s1) #             if t1_l == t2_l: #                 bad_sentences.append(t1) # # # if require_good_pair: #     df_cleaned = pd.DataFrame({"not_wrong": good_sentences, "wrong": bad_sentences}) #     df_cleaned.to_csv("cleaned_results4.tsv", sep="\t", header=None) # else: #     df_cleaned_good = pd.DataFrame({"good": good_sentences}) #     df_cleaned_wrong = pd.DataFrame({"wrong": bad_sentences}) #     df_cleaned_good.to_csv("cleaned_good_results4.tsv", sep="\t", header=None) #     df_cleaned_wrong.to_csv("cleaned_wrong_results4.tsv", sep="\t", header=None)