<reponame>dohyun1411/gpt3-sandbox<filename>action.py
import argparse, csv, copy, random, os
from api import GPT, Example, set_openai_key
set_openai_key('<KEY>')

input_prefix = "Instruction: "
input_suffix = "\n"
output_prefix = "Output: "
output_suffix = "\n\n"
append_output_prefix_to_query = False
get_reply = 1
save = 1
pos_action = "no pick up"
neg_action = "pick up"
test_size = 100
num_examples = 10
# ~30

gpt = GPT(
    input_prefix=input_prefix,
    input_suffix=input_suffix,
    output_prefix=output_prefix,
    output_suffix=output_suffix,
    append_output_prefix_to_query=append_output_prefix_to_query
)
seed_num = 716
result = f"{input_prefix = }, {input_suffix = }, {output_prefix = }, {output_suffix = },\n{append_output_prefix_to_query = }"
result += '\n' + f"{num_examples = }"
result += '\n' + f"{seed_num = }"
result += '\n--------------------\n'


create_dataset = False
if create_dataset:
    import nltk
    from nltk.corpus import worndet as wn

    keywords = ["heavy", "slippery", "brittle"]
    use_synonyms = True
    use_hypernyms = True
    use_hyponyms = True
    used_words = set(keywords)

    for keyword in keywords:
        if use_synonyms:
            pass


with open("action.txt", 'r') as f:
    keyword_group = f.read().split("\n\n")
# only_positive = True
# if only_positive:
#     keyword_group = keyword_group[:len(keyword_group) // 2]

pos_modifiers = []
neg_modifiers = []
for keywords in keyword_group[:len(keyword_group) // 2]:
    for keyword in keywords.split("\n"):
        pos_modifiers.append(keyword)
for keywords in keyword_group[len(keyword_group) // 2:]:
    for keyword in keywords.split("\n"):
        neg_modifiers.append(keyword)

actions = ["Pick up", "Give me", "Bring", "Get"]
objects = ["block", "box", "soap", "glass", "book", "ball", "cup", "orange", "apple", "ruler"]
use_normal_modifier = True
if use_normal_modifier:
    normal_modifiers = ["red", "yellow", "orange", "green", "blue", "left", "right"]
    neg_modifiers.extend(normal_modifiers)

random.seed(seed_num)
random.shuffle(pos_modifiers)
random.shuffle(neg_modifiers)
random.shuffle(actions)
random.shuffle(objects)

sentences = []
for modifier in pos_modifiers[:num_examples // 2]:
    sentence = random.choice(actions) + " the " + modifier + " " + random.choice(objects) + "::" + pos_action
    sentences.append(sentence)
for modifier in neg_modifiers[:num_examples // 2]:
    sentence = random.choice(actions) + " the " + modifier + " " + random.choice(objects) + "::" + neg_action
    sentences.append(sentence)
# result += "\n".join(sentences)
# result += '\n--------------------\n'


for sentence in sentences:
    input, output = sentence.split('::')
    gpt.add_example(Example(input, output))
result += gpt.craft_query("Pick up the red block (sample)")
result += '\n--------------------\n'

pos_correct, pos_wrong = 0, 0
neg_correct, neg_wrong = 0, 0
if get_reply:
    for _ in range(test_size // 2):
        prompt = random.choice(actions) + " the " + random.choice(pos_modifiers[num_examples // 2:]) + " " + random.choice(objects)
        reply = gpt.get_top_reply(prompt)
        if pos_action in reply:
            pos_correct += 1
        else:
            pos_wrong += 1
        result += f'({pos_action in reply})\n' + prompt + '\n' + reply
        prompt = random.choice(actions) + " the " + random.choice(neg_modifiers[num_examples // 2:]) + " " + random.choice(objects)
        reply = gpt.get_top_reply(prompt)
        if neg_action in reply:
            neg_correct += 1
        else:
            neg_wrong += 1
        result += f'({neg_action in reply})\n' + prompt + '\n' + reply
correct = pos_correct + neg_correct
try:
    acc = f"{pos_action}: {pos_correct} / {pos_correct + pos_wrong} = {pos_correct * 100 / (pos_correct + pos_wrong):.2f}\n"
    acc += f"{neg_action}: {neg_correct} / {neg_correct + neg_wrong} = {neg_correct * 100 / (neg_correct + neg_wrong):.2f}\n"
    acc += f"{correct} / {test_size} = {correct * 100 / (test_size):.2f}\n"
    result = acc + result + acc
except ZeroDivisionError:
    pass
print(result)

n = sorted(map(lambda x: int(x.split('.')[0]), os.listdir('results')))[-1] + 1
# n = 3
if save:
    print(f"Save as results/{n}.txt")
    with open(f"results/{n}.txt", 'w') as f:
        f.write(result)
import argparse, csv, copy, random, os
from api import GPT, Example, set_openai_key
set_openai_key('<KEY>')

input_prefix = "Instruction: "
input_suffix = "\n"
output_prefix = "Output: "
output_suffix = "\n\n"
append_output_prefix_to_query = False
get_reply = 1
save = 1
pos_action = "no pick up"
neg_action = "pick up"
test_size = 100
num_examples = 10
# ~30

gpt = GPT(
    input_prefix=input_prefix,
    input_suffix=input_suffix,
    output_prefix=output_prefix,
    output_suffix=output_suffix,
    append_output_prefix_to_query=append_output_prefix_to_query
)
seed_num = 716
result = f"{input_prefix = }, {input_suffix = }, {output_prefix = }, {output_suffix = },\n{append_output_prefix_to_query = }"
result += '\n' + f"{num_examples = }"
result += '\n' + f"{seed_num = }"
result += '\n--------------------\n'


create_dataset = False
if create_dataset:
    import nltk
    from nltk.corpus import worndet as wn

    keywords = ["heavy", "slippery", "brittle"]
    use_synonyms = True
    use_hypernyms = True
    use_hyponyms = True
    used_words = set(keywords)

    for keyword in keywords:
        if use_synonyms:
            pass


with open("action.txt", 'r') as f:
    keyword_group = f.read().split("\n\n")
# only_positive = True
# if only_positive:
#     keyword_group = keyword_group[:len(keyword_group) // 2]

pos_modifiers = []
neg_modifiers = []
for keywords in keyword_group[:len(keyword_group) // 2]:
    for keyword in keywords.split("\n"):
        pos_modifiers.append(keyword)
for keywords in keyword_group[len(keyword_group) // 2:]:
    for keyword in keywords.split("\n"):
        neg_modifiers.append(keyword)

actions = ["Pick up", "Give me", "Bring", "Get"]
objects = ["block", "box", "soap", "glass", "book", "ball", "cup", "orange", "apple", "ruler"]
use_normal_modifier = True
if use_normal_modifier:
    normal_modifiers = ["red", "yellow", "orange", "green", "blue", "left", "right"]
    neg_modifiers.extend(normal_modifiers)

random.seed(seed_num)
random.shuffle(pos_modifiers)
random.shuffle(neg_modifiers)
random.shuffle(actions)
random.shuffle(objects)

sentences = []
for modifier in pos_modifiers[:num_examples // 2]:
    sentence = random.choice(actions) + " the " + modifier + " " + random.choice(objects) + "::" + pos_action
    sentences.append(sentence)
for modifier in neg_modifiers[:num_examples // 2]:
    sentence = random.choice(actions) + " the " + modifier + " " + random.choice(objects) + "::" + neg_action
    sentences.append(sentence)
# result += "\n".join(sentences)
# result += '\n--------------------\n'


for sentence in sentences:
    input, output = sentence.split('::')
    gpt.add_example(Example(input, output))
result += gpt.craft_query("Pick up the red block (sample)")
result += '\n--------------------\n'

pos_correct, pos_wrong = 0, 0
neg_correct, neg_wrong = 0, 0
if get_reply:
    for _ in range(test_size // 2):
        prompt = random.choice(actions) + " the " + random.choice(pos_modifiers[num_examples // 2:]) + " " + random.choice(objects)
        reply = gpt.get_top_reply(prompt)
        if pos_action in reply:
            pos_correct += 1
        else:
            pos_wrong += 1
        result += f'({pos_action in reply})\n' + prompt + '\n' + reply
        prompt = random.choice(actions) + " the " + random.choice(neg_modifiers[num_examples // 2:]) + " " + random.choice(objects)
        reply = gpt.get_top_reply(prompt)
        if neg_action in reply:
            neg_correct += 1
        else:
            neg_wrong += 1
        result += f'({neg_action in reply})\n' + prompt + '\n' + reply
correct = pos_correct + neg_correct
try:
    acc = f"{pos_action}: {pos_correct} / {pos_correct + pos_wrong} = {pos_correct * 100 / (pos_correct + pos_wrong):.2f}\n"
    acc += f"{neg_action}: {neg_correct} / {neg_correct + neg_wrong} = {neg_correct * 100 / (neg_correct + neg_wrong):.2f}\n"
    acc += f"{correct} / {test_size} = {correct * 100 / (test_size):.2f}\n"
    result = acc + result + acc
except ZeroDivisionError:
    pass
print(result)

n = sorted(map(lambda x: int(x.split('.')[0]), os.listdir('results')))[-1] + 1
# n = 3
if save:
    print(f"Save as results/{n}.txt")
    with open(f"results/{n}.txt", 'w') as f:
        f.write(result)
# ~30 # only_positive = True # if only_positive: #     keyword_group = keyword_group[:len(keyword_group) // 2] # result += "\n".join(sentences) # result += '\n--------------------\n' # n = 3