# Import necessary libraries
import gradio as gr
import numpy as np
import pandas as pd
from rapidfuzz.distance import Levenshtein, JaroWinkler
from sentence_transformers import SentenceTransformer, util
from typing import List
import zipfile
import os
import io
def calculate_similarity(code1, code2, Ws, Wl, Wj, model_name):
model = SentenceTransformer(model_name)
embedding1 = model.encode(code1)
embedding2 = model.encode(code2)
sim_similarity = util.cos_sim(embedding1, embedding2).item()
lev_ratio = Levenshtein.normalized_similarity(code1, code2)
jaro_winkler_ratio = JaroWinkler.normalized_similarity(code1, code2)
overall_similarity = Ws * sim_similarity + Wl * lev_ratio + Wj * jaro_winkler_ratio
return "The similarity score between the two codes is: %.2f" % overall_similarity
# Define the function to process the uploaded file and return a DataFrame
def extract_and_read_compressed_file(file_path):
file_names = []
codes = []
# Handle .zip files
if file_path.endswith('.zip'):
with zipfile.ZipFile(file_path, 'r') as z:
file_names = z.namelist()
codes = [z.read(file).decode('utf-8', errors='ignore') for file in file_names]
else:
raise ValueError("Unsupported file type. Only .zip is supported.")
return file_names, codes
def filter_and_return_top(df, similarity_threshold,returned_results):
filtered_df = df[df['similarity_score'] > similarity_threshold]
return filtered_df.head(returned_results)
# Perform paraphrase mining with the specified weights
def perform_paraphrase_mining(model, codes_list, weight_semantic, weight_levenshtein, weight_jaro_winkler):
return paraphrase_mining_with_combined_score(
model,
codes_list,
weight_semantic=weight_semantic,
weight_levenshtein=weight_levenshtein,
weight_jaro_winkler=weight_jaro_winkler
)
def paraphrase_mining_with_combined_score(
model,
sentences: List[str],
show_progress_bar: bool = False,
weight_semantic: float = 1.0,
weight_levenshtein: float = 0.0,
weight_jaro_winkler: float = 0.0
):
embeddings = model.encode(
sentences, show_progress_bar=show_progress_bar, convert_to_tensor=True)
paraphrases = util.paraphrase_mining_embeddings(embeddings, score_function=util.cos_sim)
results = []
for score, i, j in paraphrases:
lev_ratio = Levenshtein.normalized_similarity(sentences[i], sentences[j])
jaro_winkler_ratio = JaroWinkler.normalized_similarity(sentences[i], sentences[j])
combined_score = (weight_semantic * score) + \
(weight_levenshtein * lev_ratio) + \
(weight_jaro_winkler * jaro_winkler_ratio)
results.append([combined_score, i, j])
results = sorted(results, key=lambda x: x[0], reverse=True)
return results
def get_sim_list(zipped_file,Ws, Wl, Wj, model_name,threshold,number_results):
file_names, codes = extract_and_read_compressed_file(zipped_file)
model = SentenceTransformer(model_name)
code_pairs = perform_paraphrase_mining(model, codes,Ws, Wl, Wj)
pairs_results = []
for score, i, j in code_pairs:
pairs_results.append({
'file_name_1': file_names[i],
'file_name_2': file_names[j],
'similarity_score': score
})
similarity_df = pd.concat([pd.DataFrame(pairs_results)], ignore_index=True)
similarity_df = similarity_df.sort_values(by='similarity_score', ascending=False)
result = filter_and_return_top(similarity_df,threshold,number_results).round(2)
return result
# Define the Gradio app
with gr.Blocks(theme=gr.themes.Glass()) as demo:
# Tab for similarity calculation
with gr.Tab("Code Pair Similarity"):
# Input components
code1 = gr.Textbox(label="Code 1")
code2 = gr.Textbox(label="Code 2")
# Accordion for weights and models
with gr.Accordion("Weights and Models", open=False):
Ws = gr.Slider(0, 1, value=0.7, label="Semantic Search Weight", step=0.1)
Wl = gr.Slider(0, 1, value=0.3, label="Levenshiern Distance Weight", step=0.1)
Wj = gr.Slider(0, 1, value=0.0, label="Jaro Winkler Weight", step=0.1)
model_dropdown = gr.Dropdown(
[("codebert", "microsoft/codebert-base"),
("graphcodebert", "microsoft/graphcodebert-base"),
("UnixCoder", "microsoft/unixcoder-base-unimodal"),
("CodeBERTa", "huggingface/CodeBERTa-small-v1"),
("CodeT5 small", "Salesforce/codet5-small"),
("PLBART", "uclanlp/plbart-java-cs"),],
label="Select Model",
value= "uclanlp/plbart-java-cs"
)
# Output component
output = gr.Textbox(label="Similarity Score")
def update_weights(Ws, Wl, Wj):
total = Ws + Wl + Wj
if total != 1:
Wj = 1 - (Ws + Wl)
return Ws, Wl, Wj
# Update weights when any slider changes
Ws.change(update_weights, [Ws, Wl, Wj], [Ws, Wl, Wj])
Wl.change(update_weights, [Ws, Wl, Wj], [Ws, Wl, Wj])
Wj.change(update_weights, [Ws, Wl, Wj], [Ws, Wl, Wj])
# Button to trigger the similarity calculation
calculate_btn = gr.Button("Calculate Similarity")
calculate_btn.click(calculate_similarity, inputs=[code1, code2, Ws, Wl, Wj, model_dropdown], outputs=output)
# Tab for file upload and DataFrame output
with gr.Tab("Code Collection Pair Similarity"):
# File uploader component
file_uploader = gr.File(label="Upload a Zip file",file_types=[".zip"])
with gr.Accordion("Weights and Models", open=False):
Ws = gr.Slider(0, 1, value=0.7, label="Semantic Search Weight", step=0.1)
Wl = gr.Slider(0, 1, value=0.3, label="Levenshiern Distance Weight", step=0.1)
Wj = gr.Slider(0, 1, value=0.0, label="Jaro Winkler Weight", step=0.1)
model_dropdown = gr.Dropdown(
[("codebert", "microsoft/codebert-base"),
("graphcodebert", "microsoft/graphcodebert-base"),
("UnixCoder", "microsoft/unixcoder-base-unimodal"),
("CodeBERTa", "huggingface/CodeBERTa-small-v1"),
("CodeT5 small", "Salesforce/codet5-small"),
("PLBART", "uclanlp/plbart-java-cs"),],
label="Select Model",
value= "uclanlp/plbart-java-cs"
)
threshold = gr.Slider(0, 1, value=0, label="Threshold", step=0.01)
number_results = gr.Slider(1, 1000, value=10, label="Number of Returned pairs", step=1)
# Output component for the DataFrame
df_output = gr.Dataframe(label="Results")
# Button to trigger the file processing
process_btn = gr.Button("Process File")
process_btn.click(get_sim_list, inputs=[file_uploader, Ws, Wl, Wj, model_dropdown,threshold,number_results], outputs=df_output)
# Launch the Gradio app with live=True
demo.launch(show_error=True,debug=True)