start questions task

.gitattributes

@@ -34,4 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 *.json filter=lfs diff=lfs merge=lfs -text
 dialogues_set/* filter=lfs diff=lfs merge=lfs -text
-dialogues_set/*.json filter=lfs diff=lfs merge=lfs -text
+questions/* filter=lfs diff=lfs merge=lfs -text

app.py

@@ -46,21 +46,16 @@ See https://github.com/openai/openai-python/blob/main/chatml.md for information
 
 def read_qs():
     qs, qas = [], []
-    directory = "./dialogues_set"
+    directory = "./questions"
     filenames = [
-        'dialogues_film.json', 
-        'dialogues_jindong.json', 
-        'dialogues_music.json', 
-        'dialogues_natural.json', 
-        'dialogues_taobao.json', 
-        'dialogues_travel_kd.json'
+        'math_question.json', 
+        'qa_question.txt', 
+        'summarization_question.txt',
     ]
     for filename in filenames:
         with open(f"{directory}/{filename}", "r", encoding="utf-8") as f:
             for idx,line in enumerate(f):
-                idx2query = json.loads(line)
-                query = idx2query[str(idx)]
-                qs.append(query)
+                qs.append(line.replace("生成摘要","生成中文摘要"))
     print(f"read {len(qs)} queries from files")
     
     if os.path.exists(f"{directory}/qas.json"):

questions/math_question.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4825026285cfa090fff557b62277a3fa2b7de054c68e82069f89ff69425e64e6
+size 2694259

questions/qa_question.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f17826d3daa4c539c461f4df4fbdc360f565a71ab46feef449b5ee231f44ccba
+size 937034

questions/summarization_question.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e468e0b18f2c906f998e12e31dfc6f4e6bee92526dbb892df0a68e517e1e0aa1
+size 156823444