|
import json |
|
import requests |
|
import tiktoken |
|
import logging |
|
from difflib import Differ |
|
|
|
|
|
def json_validator( |
|
text: str, |
|
openai_key: str, |
|
retry: int = 3, |
|
model: str = "text-davinci-003" |
|
): |
|
|
|
encoder = tiktoken.encoding_for_model(model) |
|
|
|
for _ in range(retry): |
|
try: |
|
return json.loads(text) |
|
except Exception: |
|
|
|
|
|
prompt = f"Modify the following into a valid json format:\n{text}" |
|
prompt_token_length = len(encoder.encode(prompt)) |
|
|
|
data = { |
|
"model": model, |
|
"prompt": prompt, |
|
"max_tokens": 4097 - prompt_token_length - 64 |
|
} |
|
headers = { |
|
"Content-Type": "application/json", |
|
"Authorization": f"Bearer {openai_key}" |
|
} |
|
for _ in range(retry): |
|
response = requests.post( |
|
'https://api.openai.com/v1/completions', |
|
json=data, |
|
headers=headers, |
|
timeout=300 |
|
) |
|
if response.status_code != 200: |
|
logging.warning(f'fetch openai chat retry: {response.text}') |
|
continue |
|
text = response.json()['choices'][0]['text'] |
|
break |
|
|
|
if response.status_code != 200: |
|
raise Exception(response.json()['error']) |
|
|
|
return text |
|
|
|
|
|
def fetch_chat( |
|
prompt: str, |
|
openai_key: str, |
|
retry: int = 3, |
|
model: str = "gpt-3.5-turbo-16k" |
|
): |
|
data = { |
|
"model": model, |
|
"messages": [{"role": "user", "content": prompt}] |
|
} |
|
headers = { |
|
"Content-Type": "application/json", |
|
"Authorization": f"Bearer {openai_key}" |
|
} |
|
for _ in range(retry): |
|
response = requests.post( |
|
'https://api.openai.com/v1/chat/completions', |
|
json=data, |
|
headers=headers, |
|
timeout=300 |
|
) |
|
if response.status_code != 200: |
|
logging.warning(f'fetch openai chat retry: {response.text}') |
|
continue |
|
result = response.json()['choices'][0]['message']['content'] |
|
return result |
|
|
|
return response.json()["error"] |
|
|
|
|
|
def diff_texts(text1: str, text2: str) -> list: |
|
d = Differ() |
|
return [ |
|
(token[2:], token[0] if token[0] != " " else None) |
|
for token in d.compare(text1, text2) |
|
] |
|
|