update

Dockerfile

@@ -5,7 +5,7 @@ WORKDIR /code
 COPY . /code
 
 RUN apt-get update
-RUN apt-get install -y wget unzip ffmpeg build-essential git
+RUN apt-get install -y wget unzip ffmpeg build-essential git git-lfs
 
 RUN pip install --upgrade pip
 RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt

data/eval_data/byteplus/byteplus/seed-1-6-flash-250615/shenzhen_sase/byteplus_api_key/20250728_113641/agent-bingoplus-ph-200-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a574d56126be957ef4d283af06243125886f7544ccaa5bbbe0b01900abe2c62f
+size 2417697

data/eval_data/byteplus/byteplus/seed-1-6-flash-250615/shenzhen_sase/byteplus_api_key/20250728_113641/agent-bingoplus-ph-90-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:118787cf7fd66a6683864ff4b79fc648c7d17c65b420c25092c14857c75674ed
+size 258515

data/eval_data/gemini_google/google/llama-4-maverick-17b-128e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-bingoplus-ph-200-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab42fc8b853062a9391db33fe890869e7f61e7f9c118ea2c84e3c3555768ca00
+size 2419510

data/eval_data/gemini_google/google/llama-4-maverick-17b-128e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-bingoplus-ph-90-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d738dbb5fa0aef7cc3880b0ec50f2a54143ce586b74bb3c1cffe009f53344dc
+size 258673

data/eval_data/gemini_google/google/llama-4-maverick-17b-128e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-lingoace-zh-400-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ae30069ee95459c290f53eb50dcb72cb2c11a8a7c3691a96006f4d462dd767b
+size 1211487

data/eval_data/gemini_google/google/llama-4-maverick-17b-128e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-lingoace-zh-80-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ea3a2b7e5c28a98464352433baecdb7f6c011046d6853282709f7b62ca1386c
+size 874387

data/eval_data/gemini_google/google/llama-4-maverick-17b-128e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/arc-easy-1000-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:970ffc784ca83d2ce6e826d3303590d0646f77395bdd832fa809cf09dad46529
+size 720927

data/eval_data/gemini_google/google/llama-4-scout-17b-16e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-bingoplus-ph-200-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f762c204ac2438aebe08f143bbffddd10d2e94701dd787b103506c09c79f1c1b
+size 2471787

data/eval_data/gemini_google/google/llama-4-scout-17b-16e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-bingoplus-ph-90-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6963aa07be72dff967b2388cb4d0303ed76624ba7b48f3f5861c9b207c08448
+size 258578

data/eval_data/gemini_google/google/llama-4-scout-17b-16e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-lingoace-zh-400-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b38cb68452d6f237d275aa03a6c589ece653d4f8ecd5e808d41bb0ac729d850
+size 1211826

data/eval_data/gemini_google/google/llama-4-scout-17b-16e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-lingoace-zh-80-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40aab0bb0dd05948d878e0ffab0cb84eca630530079619bd79744957cf42bef2
+size 874346

examples/make_raw_dataset/step_3_filter_by_keywords.py

@@ -50,12 +50,29 @@ def main():
 
             for key_str in [
                 # "BingoPlus",
-                " COD ",
+                # " COD ",
+                # "NXPay",
+                # "NX Money",
+                # "Exodus Telecom",
+                # "Exodus Retail",
+                "Exodus Automotive",
+                # "kta kilat", "KTA KILAT",
+                # "NXCloud",
+                # "作为VIP客户",
+                "FedEx",
             ]:
                 if system_prompt.__contains__(key_str) or user_prompt.__contains__(key_str):
                     print(f"process: {sample_dir.as_posix()}")
                     # tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-bingoplus"
-                    tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-cod"
+                    # tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-cod"
+                    # tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-nxpay"
+                    # tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-nxmoney"
+                    # tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-exodus-retail"
+                    # tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-exodus-automotive"
+                    # tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-kta"
+                    # tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-nxcloud"
+                    # tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-vip"
+                    tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-fedex"
                     tgt_dir.mkdir(parents=True, exist_ok=True)
                     shutil.move(
                         sample_dir.as_posix(),

examples/test_metrics/bingoplus_chat_metric.py

@@ -38,12 +38,12 @@ python3 azure_openai.py --model_name gpt-4o-mini \
     )
     parser.add_argument(
         "--eval_data_file",
-        default=(project_path / "data/eval_data/gemini_google/google/gemini-2.5-flash-lite-preview-06-17/shenzhen_sase/google_potent_veld_462405_t3/20250729_161543/agent-bingoplus-ph-200-chat.jsonl.raw").as_posix(),
+        default=(project_path / "data/eval_data/gemini_google/google/llama-4-scout-17b-16e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-bingoplus-ph-200-chat.jsonl.raw").as_posix(),
         type=str
     )
     parser.add_argument(
         "--output_file",
-        default=(project_path / "data/eval_data/gemini_google/google/gemini-2.5-flash-lite-preview-06-17/shenzhen_sase/google_potent_veld_462405_t3/20250729_161543/agent-bingoplus-ph-200-chat.jsonl").as_posix(),
+        default=(project_path / "data/eval_data/gemini_google/google/llama-4-scout-17b-16e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-bingoplus-ph-200-chat.jsonl").as_posix(),
         type=str
     )
     parser.add_argument(

examples/test_metrics/lingoace_chat_metric.py

@@ -43,12 +43,12 @@ python3 azure_openai.py --model_name gpt-4o-mini \
     )
     parser.add_argument(
         "--eval_data_file",
-        default=(project_path / "data/eval_data/byteplus/byteplus/seed-1-6-flash-250615/shenzhen_sase/byteplus_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl.raw").as_posix(),
+        default=(project_path / "data/eval_data/gemini_google/google/llama-4-scout-17b-16e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-lingoace-zh-80-chat.jsonl.raw").as_posix(),
         type=str
     )
     parser.add_argument(
         "--output_file",
-        default=(project_path / "data/eval_data/byteplus/byteplus/seed-1-6-flash-250615/shenzhen_sase/byteplus_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl").as_posix(),
+        default=(project_path / "data/eval_data/gemini_google/google/llama-4-scout-17b-16e-instruct-maas/shenzhen_sase/google_potent_veld_462405_t3/20250731_162116/agent-lingoace-zh-80-chat.jsonl").as_posix(),
         type=str
     )
     parser.add_argument(

llm_eval_script/byteplus.py

@@ -49,8 +49,9 @@ def get_args():
     )
     parser.add_argument(
         "--eval_dataset_name",
+        default="agent-bingoplus-ph-90-choice.jsonl",
         # default="agent-lingoace-zh-400-choice.jsonl",
-        default="arc-easy-1000-choice.jsonl",
+        # default="arc-easy-1000-choice.jsonl",
         type=str
     )
     parser.add_argument(

llm_eval_script/byteplus_chat.py

@@ -42,14 +42,15 @@ def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--model_name",
-        default="seed-1-6-250615",
-        # default="seed-1-6-flash-250615",
+        # default="seed-1-6-250615",
+        default="seed-1-6-flash-250615",
         # default="deepseek-v3-250324",
         type=str
     )
     parser.add_argument(
         "--eval_dataset_name",
-        default="agent-lingoace-zh-80-chat.jsonl",
+        # default="agent-lingoace-zh-80-chat.jsonl",
+        default="agent-bingoplus-ph-200-chat.jsonl",
         type=str
     )
     parser.add_argument(

llm_eval_script/gemini_google.py

@@ -1,5 +1,25 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
+"""
+https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude?hl=zh-cn
+https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude?hl=zh-cn
+
+
+Llama
+
+https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/llama/use-llama?hl=zh-cn
+https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/llama/use-llama?hl=zh-cn#regions-quotas
+
+Model Name
+llama-4-maverick-17b-128e-instruct-maas
+llama-4-scout-17b-16e-instruct-maas
+
+区域选择 us-east5
+
+
+
+
+"""
 import argparse
 from datetime import datetime
 import json
@@ -25,13 +45,17 @@ def get_args():
         "--model_name",
         # default="gemini-2.5-pro",   # The model does not support setting thinking_budget to 0.
         # default="gemini-2.5-flash",
-        default="gemini-2.5-flash-lite-preview-06-17",
+        # default="gemini-2.5-flash-lite-preview-06-17",
+        # default="claude-opus-4@20250514",
+        # default="claude-sonnet-4@20250514",
+        # default="llama-4-maverick-17b-128e-instruct-maas",
+        default="llama-4-scout-17b-16e-instruct-maas",
         type=str
     )
     parser.add_argument(
         "--eval_dataset_name",
-        default="agent-bingoplus-ph-90-choice.jsonl",
-        # default="agent-lingoace-zh-400-choice.jsonl",
+        # default="agent-bingoplus-ph-90-choice.jsonl",
+        default="agent-lingoace-zh-400-choice.jsonl",
         # default="arc-easy-1000-choice.jsonl",
         type=str
     )
@@ -55,6 +79,17 @@ def get_args():
         default="google_potent_veld_462405_t3",
         type=str
     )
+    parser.add_argument(
+        "--create_time_str",
+        # default="null",
+        default="20250731_162116",
+        type=str
+    )
+    parser.add_argument(
+        "--interval",
+        default=1,
+        type=int
+    )
     args = parser.parse_args()
     return args
 
@@ -79,9 +114,13 @@ def main():
     eval_data_dir = Path(args.eval_data_dir)
     eval_data_dir.mkdir(parents=True, exist_ok=True)
 
-    tz = ZoneInfo("Asia/Shanghai")
-    now = datetime.now(tz)
-    create_time_str = now.strftime("%Y%m%d_%H%M%S")
+    if args.create_time_str == "null":
+        tz = ZoneInfo("Asia/Shanghai")
+        now = datetime.now(tz)
+        create_time_str = now.strftime("%Y%m%d_%H%M%S")
+        # create_time_str = "20250729-interval-5"
+    else:
+        create_time_str = args.create_time_str
 
     eval_dataset = eval_dataset_dir / args.eval_dataset_name
 
@@ -91,7 +130,8 @@ def main():
     client = genai.Client(
         vertexai=True,
         project=project_id,
-        location="global",
+        # location="global",
+        location="us-east5",
     )
     generate_content_config = types.GenerateContentConfig(
         top_p=0.95,
@@ -137,6 +177,8 @@ def main():
                     ]
                 )
             ]
+            time.sleep(args.interval)
+            print(f"sleep: {args.interval}")
             time_begin = time.time()
             llm_response: types.GenerateContentResponse = client.models.generate_content(
                 model=args.model_name,

llm_eval_script/gemini_google_chat.py

@@ -25,7 +25,9 @@ def get_args():
         "--model_name",
         # default="gemini-2.5-pro",   # The model does not support setting thinking_budget to 0.
         # default="gemini-2.5-flash",
-        default="gemini-2.5-flash-lite-preview-06-17",
+        # default="gemini-2.5-flash-lite-preview-06-17",
+        # default="llama-4-maverick-17b-128e-instruct-maas",
+        default="llama-4-scout-17b-16e-instruct-maas",
         type=str
     )
     parser.add_argument(
@@ -57,12 +59,12 @@ def get_args():
     parser.add_argument(
         "--create_time_str",
         # default="null",
-        default="20250729_161543",
+        default="20250731_162116",
         type=str
     )
     parser.add_argument(
         "--interval",
-        default=5,
+        default=1,
         type=int
     )
     args = parser.parse_args()
@@ -105,7 +107,9 @@ def main():
     client = genai.Client(
         vertexai=True,
         project=project_id,
-        location="global",
+        # location="global",
+        location="us-east5",
+
     )
     generate_content_config = types.GenerateContentConfig(
         top_p=0.95,

main.py

@@ -17,6 +17,7 @@ docker run -itd \
 --name llm_eval_system_7862 \
 --restart=always \
 --network host \
+-v /data/tianxing/PycharmProjects/llm_eval_system:/data/tianxing/PycharmProjects/llm_eval_system \
 python:3.12 \
 /bin/bash