__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
import os
import io
import gradio as gr
import pandas as pd
import json
import shutil
import tempfile
import datetime
import zipfile
import numpy as np
from constants import *
from huggingface_hub import Repository
HF_TOKEN = os.environ.get("HF_TOKEN")
global data_component, filter_component
def upload_file(files):
file_paths = [file.name for file in files]
return file_paths
def add_new_eval(
input_file,
model_name_textbox: str,
revision_name_textbox: str,
model_link: str,
team_name: str,
contact_email: str,
access_type: str,
model_publish: str,
model_resolution: str,
model_fps: str,
model_frame: str,
model_video_length: str,
model_checkpoint: str,
model_commit_id: str,
model_video_format: str
):
if input_file is None:
return "Error! Empty file!"
if model_link == '' or model_name_textbox == '' or contact_email == '':
return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
# upload_data=json.loads(input_file)
upload_content = input_file
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
filename = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
now = datetime.datetime.now()
update_time = now.strftime("%Y-%m-%d") # Capture update time
with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f:
f.write(input_file)
# shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}"))
csv_data = pd.read_csv(CSV_DIR)
if revision_name_textbox == '':
col = csv_data.shape[0]
model_name = model_name_textbox.replace(',',' ')
else:
model_name = revision_name_textbox.replace(',',' ')
model_name_list = csv_data['Model Name (clickable)']
name_list = [name.split(']')[0][1:] for name in model_name_list]
if revision_name_textbox not in name_list:
col = csv_data.shape[0]
else:
col = name_list.index(revision_name_textbox)
if model_link == '':
model_name = model_name # no url
else:
model_name = '[' + model_name + '](' + model_link + ')'
os.makedirs(filename, exist_ok=True)
with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref:
zip_ref.extractall(filename)
upload_data = {}
for file in os.listdir(filename):
if file.startswith('.') or file.startswith('__'):
print(f"Skip the file: {file}")
continue
cur_file = os.path.join(filename, file)
if os.path.isdir(cur_file):
for subfile in os.listdir(cur_file):
if subfile.endswith(".json"):
with open(os.path.join(cur_file, subfile)) as ff:
cur_json = json.load(ff)
print(file, type(cur_json))
if isinstance(cur_json, dict):
print(cur_json.keys())
for key in cur_json:
upload_data[key.replace('_',' ')] = cur_json[key][0]
print(f"{key}:{cur_json[key][0]}")
elif cur_file.endswith('json'):
with open(cur_file) as ff:
cur_json = json.load(ff)
print(file, type(cur_json))
if isinstance(cur_json, dict):
print(cur_json.keys())
for key in cur_json:
upload_data[key.replace('_',' ')] = cur_json[key][0]
print(f"{key}:{cur_json[key][0]}")
# add new data
new_data = [model_name]
print('upload_data:', upload_data)
for key in TASK_INFO:
if key in upload_data:
new_data.append(upload_data[key])
else:
new_data.append(0)
if team_name =='' or 'vbench' in team_name.lower():
new_data.append("User Upload")
else:
new_data.append(team_name)
new_data.append(contact_email.replace(',',' and ')) # Add contact email [private]
new_data.append(update_time) # Add the update time
new_data.append(team_name)
new_data.append(access_type)
csv_data.loc[col] = new_data
csv_data = csv_data.to_csv(CSV_DIR, index=False)
with open(INFO_DIR,'a') as f:
f.write(f"{model_name}\t{update_time}\t{model_publish}\t{model_resolution}\t{model_fps}\t{model_frame}\t{model_video_length}\t{model_checkpoint}\t{model_commit_id}\t{model_video_format}\n")
submission_repo.push_to_hub()
print("success update", model_name)
return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
def add_new_eval_i2v(
input_file,
model_name_textbox: str,
revision_name_textbox: str,
model_link: str,
team_name: str,
contact_email: str,
access_type: str,
model_publish: str,
model_resolution: str,
model_fps: str,
model_frame: str,
model_video_length: str,
model_checkpoint: str,
model_commit_id: str,
model_video_format: str
):
COLNAME2KEY={
"Video-Text Camera Motion":"camera_motion",
"Video-Image Subject Consistency": "i2v_subject",
"Video-Image Background Consistency": "i2v_background",
"Subject Consistency": "subject_consistency",
"Background Consistency": "background_consistency",
"Motion Smoothness": "motion_smoothness",
"Dynamic Degree": "dynamic_degree",
"Aesthetic Quality": "aesthetic_quality",
"Imaging Quality": "imaging_quality",
"Temporal Flickering": "temporal_flickering"
}
if input_file is None:
return "Error! Empty file!"
if model_link == '' or model_name_textbox == '' or contact_email == '':
return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
upload_content = input_file
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
filename = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
now = datetime.datetime.now()
update_time = now.strftime("%Y-%m-%d") # Capture update time
with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f:
f.write(input_file)
# shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}"))
csv_data = pd.read_csv(I2V_DIR)
if revision_name_textbox == '':
col = csv_data.shape[0]
model_name = model_name_textbox.replace(',',' ')
else:
model_name = revision_name_textbox.replace(',',' ')
model_name_list = csv_data['Model Name (clickable)']
name_list = [name.split(']')[0][1:] for name in model_name_list]
if revision_name_textbox not in name_list:
col = csv_data.shape[0]
else:
col = name_list.index(revision_name_textbox)
if model_link == '':
model_name = model_name # no url
else:
model_name = '[' + model_name + '](' + model_link + ')'
os.makedirs(filename, exist_ok=True)
with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref:
zip_ref.extractall(filename)
upload_data = {}
for file in os.listdir(filename):
if file.startswith('.') or file.startswith('__'):
print(f"Skip the file: {file}")
continue
cur_file = os.path.join(filename, file)
if os.path.isdir(cur_file):
for subfile in os.listdir(cur_file):
if subfile.endswith(".json"):
with open(os.path.join(cur_file, subfile)) as ff:
cur_json = json.load(ff)
print(file, type(cur_json))
if isinstance(cur_json, dict):
print(cur_json.keys())
for key in cur_json:
upload_data[key] = cur_json[key][0]
print(f"{key}:{cur_json[key][0]}")
elif cur_file.endswith('json'):
with open(cur_file) as ff:
cur_json = json.load(ff)
print(file, type(cur_json))
if isinstance(cur_json, dict):
print(cur_json.keys())
for key in cur_json:
upload_data[key] = cur_json[key][0]
print(f"{key}:{cur_json[key][0]}")
# add new data
new_data = [model_name]
print('upload_data:', upload_data)
I2V_HEAD= ["Video-Text Camera Motion",
"Video-Image Subject Consistency",
"Video-Image Background Consistency",
"Subject Consistency",
"Background Consistency",
"Temporal Flickering",
"Motion Smoothness",
"Dynamic Degree",
"Aesthetic Quality",
"Imaging Quality" ]
for key in I2V_HEAD :
sub_key = COLNAME2KEY[key]
if sub_key in upload_data:
new_data.append(upload_data[sub_key])
else:
new_data.append(0)
if team_name =='' or 'vbench' in team_name.lower():
new_data.append("User Upload")
else:
new_data.append(team_name)
new_data.append(contact_email.replace(',',' and ')) # Add contact email [private]
new_data.append(update_time) # Add the update time
new_data.append(team_name)
new_data.append(access_type)
csv_data.loc[col] = new_data
csv_data = csv_data.to_csv(I2V_DIR , index=False)
with open(INFO_DIR,'a') as f:
f.write(f"{model_name}\t{update_time}\t{model_publish}\t{model_resolution}\t{model_fps}\t{model_frame}\t{model_video_length}\t{model_checkpoint}\t{model_commit_id}\t{model_video_format}\n")
submission_repo.push_to_hub()
print("success update", model_name)
return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
def get_normalized_df(df):
# final_score = df.drop('name', axis=1).sum(axis=1)
# df.insert(1, 'Overall Score', final_score)
normalize_df = df.copy().fillna(0.0)
for column in normalize_df.columns[1:-5]:
min_val = NORMALIZE_DIC[column]['Min']
max_val = NORMALIZE_DIC[column]['Max']
normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
return normalize_df
def get_normalized_i2v_df(df):
normalize_df = df.copy().fillna(0.0)
for column in normalize_df.columns[1:-5]:
min_val = NORMALIZE_DIC_I2V[column]['Min']
max_val = NORMALIZE_DIC_I2V[column]['Max']
normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
return normalize_df
def calculate_selected_score(df, selected_columns):
# selected_score = df[selected_columns].sum(axis=1)
selected_QUALITY = [i for i in selected_columns if i in QUALITY_LIST]
selected_SEMANTIC = [i for i in selected_columns if i in SEMANTIC_LIST]
selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_QUALITY])
selected_semantic_score = df[selected_SEMANTIC].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_SEMANTIC ])
if selected_quality_score.isna().any().any() and selected_semantic_score.isna().any().any():
selected_score = (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
return selected_score.fillna(0.0)
if selected_quality_score.isna().any().any():
return selected_semantic_score
if selected_semantic_score.isna().any().any():
return selected_quality_score
# print(selected_semantic_score,selected_quality_score )
selected_score = (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
return selected_score.fillna(0.0)
def calculate_selected_score_i2v(df, selected_columns):
# selected_score = df[selected_columns].sum(axis=1)
selected_QUALITY = [i for i in selected_columns if i in I2V_QUALITY_LIST]
selected_I2V = [i for i in selected_columns if i in I2V_LIST]
selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_QUALITY])
selected_i2v_score = df[selected_I2V].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_I2V ])
if selected_quality_score.isna().any().any() and selected_i2v_score.isna().any().any():
selected_score = (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
return selected_score.fillna(0.0)
if selected_quality_score.isna().any().any():
return selected_i2v_score
if selected_i2v_score.isna().any().any():
return selected_quality_score
# print(selected_i2v_score,selected_quality_score )
selected_score = (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
return selected_score.fillna(0.0)
def get_final_score(df, selected_columns):
normalize_df = get_normalized_df(df)
#final_score = normalize_df.drop('name', axis=1).sum(axis=1)
try:
for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1):
normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
except:
for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1):
normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
quality_score = normalize_df[QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in QUALITY_LIST])
semantic_score = normalize_df[SEMANTIC_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in SEMANTIC_LIST ])
final_score = (quality_score * QUALITY_WEIGHT + semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
if 'Total Score' in df:
df['Total Score'] = final_score
else:
df.insert(1, 'Total Score', final_score)
if 'Semantic Score' in df:
df['Semantic Score'] = semantic_score
else:
df.insert(2, 'Semantic Score', semantic_score)
if 'Quality Score' in df:
df['Quality Score'] = quality_score
else:
df.insert(3, 'Quality Score', quality_score)
selected_score = calculate_selected_score(normalize_df, selected_columns)
if 'Selected Score' in df:
df['Selected Score'] = selected_score
else:
df.insert(1, 'Selected Score', selected_score)
return df
def get_final_score_i2v(df, selected_columns):
normalize_df = get_normalized_i2v_df(df)
try:
for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1):
normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name]
except:
for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1):
normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name]
quality_score = normalize_df[I2V_QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_QUALITY_LIST])
i2v_score = normalize_df[I2V_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_LIST ])
final_score = (quality_score * I2V_QUALITY_WEIGHT + i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
if 'Total Score' in df:
df['Total Score'] = final_score
else:
df.insert(1, 'Total Score', final_score)
if 'I2V Score' in df:
df['I2V Score'] = i2v_score
else:
df.insert(2, 'I2V Score', i2v_score)
if 'Quality Score' in df:
df['Quality Score'] = quality_score
else:
df.insert(3, 'Quality Score', quality_score)
selected_score = calculate_selected_score_i2v(normalize_df, selected_columns)
if 'Selected Score' in df:
df['Selected Score'] = selected_score
else:
df.insert(1, 'Selected Score', selected_score)
# df.loc[df[9:].isnull().any(axis=1), ['Total Score', 'I2V Score']] = 'N.A.'
mask = df.iloc[:, 5:-5].isnull().any(axis=1)
df.loc[mask, ['Total Score', 'I2V Score','Selected Score' ]] = np.nan
# df.fillna('N.A.', inplace=True)
return df
def get_final_score_quality(df, selected_columns):
normalize_df = get_normalized_df(df)
for name in normalize_df.drop('Model Name (clickable)', axis=1):
normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
quality_score = normalize_df[QUALITY_TAB].sum(axis=1) / sum([DIM_WEIGHT[i] for i in QUALITY_TAB])
if 'Quality Score' in df:
df['Quality Score'] = quality_score
else:
df.insert(1, 'Quality Score', quality_score)
# selected_score = normalize_df[selected_columns].sum(axis=1) / len(selected_columns)
selected_score = normalize_df[selected_columns].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_columns])
if 'Selected Score' in df:
df['Selected Score'] = selected_score
else:
df.insert(1, 'Selected Score', selected_score)
return df
def get_baseline_df():
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
df = pd.read_csv(CSV_DIR)
df = get_final_score(df, checkbox_group.value)
df = df.sort_values(by="Selected Score", ascending=False)
present_columns = MODEL_INFO + checkbox_group.value
# print(present_columns)
df = df[present_columns]
# Add this line to display the results evaluated by VBench by default
df = df[df['Evaluated by'] == 'VBench Team']
df = convert_scores_to_percentage(df)
return df
def get_baseline_df_quality():
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
df = pd.read_csv(QUALITY_DIR)
df = get_final_score_quality(df, checkbox_group_quality.value)
df = df.sort_values(by="Selected Score", ascending=False)
present_columns = MODEL_INFO_TAB_QUALITY + checkbox_group_quality.value
df = df[present_columns]
df = convert_scores_to_percentage(df)
return df
def get_baseline_df_i2v():
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
df = pd.read_csv(I2V_DIR)
df = get_final_score_i2v(df, checkbox_group_i2v.value)
df = df.sort_values(by="Selected Score", ascending=False)
present_columns = MODEL_INFO_TAB_I2V + checkbox_group_i2v.value
# df = df[df["Sampled by"] == 'VBench Team']
df = df[present_columns]
df = convert_scores_to_percentage(df)
return df
def get_baseline_df_long():
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
df = pd.read_csv(LONG_DIR)
df = get_final_score(df, checkbox_group.value)
df = df.sort_values(by="Selected Score", ascending=False)
present_columns = MODEL_INFO + checkbox_group.value
# df = df[df["Sampled by"] == 'VBench Team']
df = df[present_columns]
df = convert_scores_to_percentage(df)
return df
def get_baseline_df_2():
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
df = pd.read_csv(VBENCH2_DIR)
# df = get_final_score(df, checkbox_group.value)
# df = df.sort_values(by="Selected Score", ascending=False)
# present_columns = MODEL_INFO + checkbox_group.value
# print(present_columns)
df = df[COLUMN_NAMES_2]
# Add this line to display the results evaluated by VBench by default
df = convert_scores_to_percentage(df)
return df
def get_all_df(selected_columns, dir=CSV_DIR):
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
df = pd.read_csv(dir)
df = get_final_score(df, selected_columns)
df = df.sort_values(by="Selected Score", ascending=False)
return df
def get_all_df_quality(selected_columns, dir=QUALITY_DIR):
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
df = pd.read_csv(dir)
df = get_final_score_quality(df, selected_columns)
df = df.sort_values(by="Selected Score", ascending=False)
return df
def get_all_df_i2v(selected_columns, dir=I2V_DIR):
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
df = pd.read_csv(dir)
df = get_final_score_i2v(df, selected_columns)
df = df.sort_values(by="Selected Score", ascending=False)
return df
def get_all_df_long(selected_columns, dir=LONG_DIR):
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
df = pd.read_csv(dir)
df = get_final_score(df, selected_columns)
df = df.sort_values(by="Selected Score", ascending=False)
return df
def get_all_df2(dir=VBENCH2_DIR):
submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
submission_repo.git_pull()
df = pd.read_csv(dir)
# df = get_final_score(df, selected_columns)
# df = df.sort_values(by="Selected Score", ascending=False)
return df
def convert_scores_to_percentage(df):
# Operate on every column in the DataFrame (except the'name 'column)
if "Sampled by" in df.columns:
skip_col =3
else:
skip_col =1
print(df)
for column in df.columns[skip_col:]: # 假设第一列是'name'
# if df[column].isdigit():
# print(df[column])
# is_numeric = pd.to_numeric(df[column], errors='coerce').notna().all()
valid_numeric_count = pd.to_numeric(df[column], errors='coerce').notna().sum()
if valid_numeric_count > 0:
df[column] = round(df[column] * 100,2)
df[column] = df[column].apply(lambda x: f"{x:05.2f}%" if pd.notna(pd.to_numeric(x, errors='coerce')) else x)
# df[column] = df[column].apply(lambda x: f"{x:05.2f}") + '%'
return df
def choose_all_quailty():
return gr.update(value=QUALITY_LIST)
def choose_all_semantic():
return gr.update(value=SEMANTIC_LIST)
def disable_all():
return gr.update(value=[])
def enable_all():
return gr.update(value=TASK_INFO)
# select function
def on_filter_model_size_method_change(selected_columns, vbench_team_sample, vbench_team_eval=False):
updated_data = get_all_df(selected_columns, CSV_DIR)
if vbench_team_sample:
updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
if vbench_team_eval:
updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team']
#print(updated_data)
# columns:
selected_columns = [item for item in TASK_INFO if item in selected_columns]
present_columns = MODEL_INFO + selected_columns
updated_data = updated_data[present_columns]
updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
updated_data = convert_scores_to_percentage(updated_data)
updated_headers = present_columns
print(COLUMN_NAMES,updated_headers,DATA_TITILE_TYPE )
update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
# print(updated_data,present_columns,update_datatype)
filter_component = gr.components.Dataframe(
value=updated_data,
headers=updated_headers,
type="pandas",
datatype=update_datatype,
interactive=False,
visible=True,
)
return filter_component#.value
def on_filter_model_size_method_change_quality(selected_columns):
updated_data = get_all_df_quality(selected_columns, QUALITY_DIR)
#print(updated_data)
# columns:
selected_columns = [item for item in QUALITY_TAB if item in selected_columns]
present_columns = MODEL_INFO_TAB_QUALITY + selected_columns
updated_data = updated_data[present_columns]
updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
updated_data = convert_scores_to_percentage(updated_data)
updated_headers = present_columns
update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
# print(updated_data,present_columns,update_datatype)
filter_component = gr.components.Dataframe(
value=updated_data,
headers=updated_headers,
type="pandas",
datatype=update_datatype,
interactive=False,
visible=True,
)
return filter_component#.value
def on_filter_model_size_method_change_i2v(selected_columns,vbench_team_sample, vbench_team_eval=False):
updated_data = get_all_df_i2v(selected_columns, I2V_DIR)
if vbench_team_sample:
updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
# if vbench_team_eval:
# updated_data = updated_data[updated_data['Eval'] == 'VBench Team']
selected_columns = [item for item in I2V_TAB if item in selected_columns]
present_columns = MODEL_INFO_TAB_I2V + selected_columns
updated_data = updated_data[present_columns]
updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
updated_data = convert_scores_to_percentage(updated_data)
updated_headers = present_columns
update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES_I2V.index(x)] for x in updated_headers]
# print(updated_data,present_columns,update_datatype)
filter_component = gr.components.Dataframe(
value=updated_data,
headers=updated_headers,
type="pandas",
datatype=update_datatype,
interactive=False,
visible=True,
)
return filter_component#.value
def on_filter_model_size_method_change_long(selected_columns, vbench_team_sample, vbench_team_eval=False):
updated_data = get_all_df_long(selected_columns, LONG_DIR)
if vbench_team_sample:
updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
if vbench_team_eval:
updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team']
selected_columns = [item for item in TASK_INFO if item in selected_columns]
present_columns = MODEL_INFO + selected_columns
updated_data = updated_data[present_columns]
updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
updated_data = convert_scores_to_percentage(updated_data)
updated_headers = present_columns
update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
filter_component = gr.components.Dataframe(
value=updated_data,
headers=updated_headers,
type="pandas",
datatype=update_datatype,
interactive=False,
visible=True,
)
return filter_component#.value
def on_filter_model_size_method_change_2(vbench_team_sample, vbench_team_eval=False):
updated_data = get_all_df(VBENCH2_DIR)
if vbench_team_sample:
updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
if vbench_team_eval:
updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team']
#print(updated_data)
# columns:
# selected_columns = [item for item in TASK_INFO if item in selected_columns]
# present_columns = MODEL_INFO + selected_columns
# updated_data = updated_data[present_columns]
# updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
# updated_data = convert_scores_to_percentage(updated_data)
updated_headers = COLUMN_NAMES_2
# print(COLUMN_NAMES,updated_headers,DATA_TITILE_TYPE )
update_datatype = VBENCH2_TITLE_TYPE
# print(updated_data,present_columns,update_datatype)
filter_component = gr.components.Dataframe(
value=updated_data,
headers=updated_headers,
type="pandas",
datatype=update_datatype,
interactive=False,
visible=True,
)
return filter_component#.value
block = gr.Blocks()
with block:
gr.Markdown(
LEADERBORAD_INTRODUCTION
)
with gr.Tabs(elem_classes="tab-buttons") as tabs:
# Table 0
with gr.TabItem("📊 VBench 1.0", elem_id="vbench-tab-table", id=1):
with gr.Row():
with gr.Accordion("Citation", open=False):
citation_button = gr.Textbox(
value=CITATION_BUTTON_TEXT,
label=CITATION_BUTTON_LABEL,
elem_id="citation-button",
lines=14,
)
gr.Markdown(
TABLE_INTRODUCTION
)
with gr.Row():
with gr.Column(scale=0.2):
choosen_q = gr.Button("Select Quality Dimensions")
choosen_s = gr.Button("Select Semantic Dimensions")
# enable_b = gr.Button("Select All")
disable_b = gr.Button("Deselect All")
with gr.Column(scale=0.8):
vbench_team_filter = gr.Checkbox(
label="Sampled by VBench Team (Uncheck to view all submissions)",
value=False,
interactive=True
)
vbench_validate_filter = gr.Checkbox(
label="Evaluated by VBench Team (Uncheck to view all submissions)",
value=True,
interactive=True
)
# selection for column part:
checkbox_group = gr.CheckboxGroup(
choices=TASK_INFO,
value=DEFAULT_INFO,
label="Evaluation Dimension",
interactive=True,
)
data_component = gr.components.Dataframe(
value=get_baseline_df,
headers=COLUMN_NAMES,
type="pandas",
datatype=DATA_TITILE_TYPE,
interactive=False,
visible=True,
height=700,
)
choosen_q.click(choose_all_quailty, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component)
choosen_s.click(choose_all_semantic, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component)
# enable_b.click(enable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter], outputs=data_component)
disable_b.click(disable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
vbench_team_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
vbench_validate_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
# VBench 2.0
with gr.TabItem("⭐ VBench 2.0", elem_id="vbench-tab-table", id=2):
with gr.Row():
with gr.Accordion("Citation", open=False):
citation_button2 = gr.Textbox(
value=CITATION_2_BUTTON_TEXT,
label=CITATION_BUTTON_LABEL,
elem_id="citation-button",
lines=14,
)
gr.Markdown(
TABLE_INTRODUCTION
)
with gr.Row():
with gr.Column():
vbench_team_filter_2 = gr.Checkbox(
label="Sampled by VBench Team (Uncheck to view all submissions)",
value=False,
interactive=True
)
vbench_validate_filter_2 = gr.Checkbox(
label="Evaluated by VBench Team (Uncheck to view all submissions)",
value=True,
interactive=True
)
data_component_2 = gr.components.Dataframe(
value=get_baseline_df_2,
headers=COLUMN_NAMES_2,
type="pandas",
datatype=VBENCH2_TITLE_TYPE,
interactive=False,
visible=True,
height=700,
)
vbench_team_filter.change(fn=on_filter_model_size_method_change_2, inputs=[vbench_team_filter_2, vbench_validate_filter], outputs=data_component_2)
vbench_validate_filter.change(fn=on_filter_model_size_method_change_2, inputs=[vbench_team_filter_2, vbench_validate_filter], outputs=data_component_2)
with gr.TabItem("Video Quality", elem_id="vbench-tab-table", id=3):
with gr.Accordion("INSTRUCTION", open=False):
citation_button = gr.Textbox(
value=QUALITY_CLAIM_TEXT,
label="",
elem_id="quality-button",
lines=2,
)
with gr.Row():
with gr.Column(scale=1.0):
# selection for column part:
checkbox_group_quality = gr.CheckboxGroup(
choices=QUALITY_TAB,
value=QUALITY_TAB,
label="Evaluation Quality Dimension",
interactive=True,
)
data_component_quality = gr.components.Dataframe(
value=get_baseline_df_quality,
headers=COLUMN_NAMES_QUALITY,
type="pandas",
datatype=DATA_TITILE_TYPE,
interactive=False,
visible=True,
)
checkbox_group_quality.change(fn=on_filter_model_size_method_change_quality, inputs=[checkbox_group_quality], outputs=data_component_quality)
# Table i2v
with gr.TabItem("VBench-I2V", elem_id="vbench-tab-table", id=4):
with gr.Accordion("NOTE", open=False):
i2v_note_button = gr.Textbox(
value=I2V_CLAIM_TEXT,
label="",
elem_id="quality-button",
lines=3,
)
with gr.Row():
with gr.Column(scale=1.0):
# selection for column part:
with gr.Row():
vbench_team_filter_i2v = gr.Checkbox(
label="Sampled by VBench Team (Uncheck to view all submissions)",
value=False,
interactive=True
)
vbench_validate_filter_i2v = gr.Checkbox(
label="Evaluated by VBench Team (Uncheck to view all submissions)",
value=False,
interactive=True
)
checkbox_group_i2v = gr.CheckboxGroup(
choices=I2V_TAB,
value=I2V_TAB,
label="Evaluation Quality Dimension",
interactive=True,
)
data_component_i2v = gr.components.Dataframe(
value=get_baseline_df_i2v,
headers=COLUMN_NAMES_I2V,
type="pandas",
datatype=I2V_TITILE_TYPE,
interactive=False,
visible=True,
)
checkbox_group_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
vbench_team_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
vbench_validate_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
with gr.TabItem("📊 VBench-Long", elem_id="vbench-tab-table", id=5):
with gr.Row():
with gr.Accordion("INSTRUCTION", open=False):
citation_button = gr.Textbox(
value=LONG_CLAIM_TEXT,
label="",
elem_id="long-ins-button",
lines=2,
)
gr.Markdown(
TABLE_INTRODUCTION
)
with gr.Row():
with gr.Column(scale=0.2):
choosen_q_long = gr.Button("Select Quality Dimensions")
choosen_s_long = gr.Button("Select Semantic Dimensions")
enable_b_long = gr.Button("Select All")
disable_b_long = gr.Button("Deselect All")
with gr.Column(scale=0.8):
with gr.Row():
vbench_team_filter_long = gr.Checkbox(
label="Sampled by VBench Team (Uncheck to view all submissions)",
value=False,
interactive=True
)
vbench_validate_filter_long = gr.Checkbox(
label="Evaluated by VBench Team (Uncheck to view all submissions)",
value=False,
interactive=True
)
checkbox_group_long = gr.CheckboxGroup(
choices=TASK_INFO,
value=DEFAULT_INFO,
label="Evaluation Dimension",
interactive=True,
)
data_component = gr.components.Dataframe(
value=get_baseline_df_long,
headers=COLUMN_NAMES,
type="pandas",
datatype=DATA_TITILE_TYPE,
interactive=False,
visible=True,
height=700,
)
choosen_q_long.click(choose_all_quailty, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
choosen_s_long.click(choose_all_semantic, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
enable_b_long.click(enable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
disable_b_long.click(disable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
checkbox_group_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
vbench_team_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
vbench_validate_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
# table info
with gr.TabItem("📝 About", elem_id="mvbench-tab-table", id=6):
gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
# table submission
with gr.TabItem("🚀 [T2V]Submit here! ", elem_id="mvbench-tab-table", id=7):
gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
with gr.Row():
gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
with gr.Row():
gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
with gr.Row():
gr.Markdown("Here is a required field", elem_classes="markdown-text")
with gr.Row():
with gr.Column():
model_name_textbox = gr.Textbox(
label="Model name", placeholder="Required field"
)
revision_name_textbox = gr.Textbox(
label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line"
)
access_type = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.")
with gr.Column():
model_link = gr.Textbox(
label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed."
)
team_name = gr.Textbox(
label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload"
)
contact_email = gr.Textbox(
label="E-Mail(Will not be displayed)", placeholder="Required field"
)
with gr.Row():
gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text")
with gr.Row():
release_time = gr.Textbox(label="Time of Publish", placeholder="1970-01-01")
model_resolution = gr.Textbox(label="resolution", placeholder="Width x Height")
model_fps = gr.Textbox(label="model fps", placeholder="FPS(int)")
model_frame = gr.Textbox(label="model frame count", placeholder="INT")
model_video_length = gr.Textbox(label="model video length", placeholder="float(2.0)")
model_checkpoint = gr.Textbox(label="model checkpoint", placeholder="optional")
model_commit_id = gr.Textbox(label="github commit id", placeholder='main')
model_video_format = gr.Textbox(label="pipeline format", placeholder='mp4')
with gr.Column():
input_file = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary')
submit_button = gr.Button("Submit Eval")
submit_succ_button = gr.Markdown("Submit Success! Please press refresh and return to LeaderBoard!", visible=False)
fail_textbox = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text",visible=False)
submission_result = gr.Markdown()
submit_button.click(
add_new_eval,
inputs = [
input_file,
model_name_textbox,
revision_name_textbox,
model_link,
team_name,
contact_email,
release_time,
access_type,
model_resolution,
model_fps,
model_frame,
model_video_length,
model_checkpoint,
model_commit_id,
model_video_format
],
outputs=[submit_button, submit_succ_button, fail_textbox]
)
with gr.TabItem("🚀 [I2V]Submit here! ", elem_id="mvbench-i2v-tab-table", id=8):
gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
with gr.Row():
gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
with gr.Row():
gr.Markdown("# ✉️✨ Submit your i2v model evaluation json file here!", elem_classes="markdown-text")
with gr.Row():
gr.Markdown("Here is a required field", elem_classes="markdown-text")
with gr.Row():
with gr.Column():
model_name_textbox_i2v = gr.Textbox(
label="Model name", placeholder="Required field"
)
revision_name_textbox_i2v = gr.Textbox(
label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line"
)
access_type_i2v = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.")
with gr.Column():
model_link_i2v = gr.Textbox(
label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed."
)
team_name_i2v = gr.Textbox(
label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload"
)
contact_email_i2v = gr.Textbox(
label="E-Mail(Will not be displayed)", placeholder="Required field"
)
with gr.Row():
gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text")
with gr.Row():
release_time_i2v = gr.Textbox(label="Time of Publish", placeholder="1970-01-01")
model_resolution_i2v = gr.Textbox(label="resolution", placeholder="Width x Height")
model_fps_i2v = gr.Textbox(label="model fps", placeholder="FPS(int)")
model_frame_i2v = gr.Textbox(label="model frame count", placeholder="INT")
model_video_length_i2v = gr.Textbox(label="model video length", placeholder="float(2.0)")
model_checkpoint_i2v = gr.Textbox(label="model checkpoint", placeholder="optional")
model_commit_id_i2v = gr.Textbox(label="github commit id", placeholder='main')
model_video_format_i2v = gr.Textbox(label="pipeline format", placeholder='mp4')
with gr.Column():
input_file_i2v = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary')
submit_button_i2v = gr.Button("Submit Eval")
submit_succ_button_i2v = gr.Markdown("Submit Success! Please press refresh and retfurn to LeaderBoard!", visible=False)
fail_textbox_i2v = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text",visible=False)
submission_result_i2v = gr.Markdown()
submit_button_i2v.click(
add_new_eval_i2v,
inputs = [
input_file_i2v,
model_name_textbox_i2v,
revision_name_textbox_i2v,
model_link_i2v,
team_name_i2v,
contact_email_i2v,
release_time_i2v,
access_type_i2v,
model_resolution_i2v,
model_fps_i2v,
model_frame_i2v,
model_video_length_i2v,
model_checkpoint_i2v,
model_commit_id_i2v,
model_video_format_i2v
],
outputs=[submit_button_i2v, submit_succ_button_i2v, fail_textbox_i2v]
)
def refresh_data():
value1 = get_baseline_df()
return value1
with gr.Row():
data_run = gr.Button("Refresh")
data_run.click(on_filter_model_size_method_change, inputs=[checkbox_group], outputs=data_component)
block.launch()