import gradio as gr from pyvis.network import Network import networkx as nx import numpy as np import pandas as pd import os from datasets import load_dataset from datasets import Features from datasets import Value from datasets import Dataset import matplotlib.pyplot as plt import re pattern = r'"(.*?)"' # this pattern captures anything in a double quotes. Secret_token = os.getenv('HF_token') dataset = load_dataset('FDSRashid/hadith_info',data_files = 'Basic_Edge_Information.csv', token = Secret_token, split = 'train') edge_info = dataset.to_pandas() features = Features({'Rawi ID': Value('int32'), 'Famous Name': Value('string'), 'Narrator Rank': Value('string'), 'Number of Narrations': Value('string'), 'Generation': Value('string')}) narrator_bios = load_dataset("FDSRashid/hadith_info", data_files = 'Teacher_Bios.csv', token = Secret_token,features=features ) narrator_bios = narrator_bios['train'].to_pandas() narrator_bios.loc[49845, 'Narrator Rank'] = 'رسول الله' narrator_bios.loc[49845, 'Number of Narrations'] = 0 narrator_bios['Number of Narrations'] = narrator_bios['Number of Narrations'].astype(int) narrator_bios.loc[49845, 'Number of Narrations'] = 327512 # 8125 Narrators have no Generation, listed in dataset as None narrator_bios['Generation'] = narrator_bios['Generation'].replace([None], [-1]) narrator_bios['Generation'] = narrator_bios['Generation'].astype(int) features = Features({'matn': Value('string'), 'taraf_ID': Value('string'), 'bookid_hadithid': Value('string')}) dataset = load_dataset("FDSRashid/hadith_info", data_files = 'All_Matns.csv',token = Secret_token, features = features) matn_info = dataset['train'].to_pandas() matn_info = matn_info.drop(97550) matn_info = matn_info.drop(307206) matn_info['taraf_ID'] = matn_info['taraf_ID'].replace('KeyAbsent', -1) matn_info['taraf_ID'] = matn_info['taraf_ID'].astype(int) # Isnad Info Hadiths column is structured like {"BookNum_HadithNum", ...} for each edge isnad_info = load_dataset('FDSRashid/hadith_info',token = Secret_token, data_files = 'isnad_info.csv', split = 'train').to_pandas() isnad_info['Hadiths Cleaned'] = isnad_info['Hadiths'].apply(lambda x: [re.findall(pattern, string)[0].split("_") for string in x[1:-1].split(',')]) # Hadiths Cleaned is a list of lists, each sub-list is Book Id, Hadith ID taraf_max = np.max(matn_info['taraf_ID'].unique()) isnad_info['Tarafs Cleaned'] = isnad_info['Tarafs'].apply(lambda x: np.array([int(i.strip(' ')) for i in x[1:-1].split(',')])) cmap = plt.colormaps['cool'] books = load_dataset('FDSRashid/Hadith_info', data_files='Books.csv', token = Secret_token)['train'].to_pandas() matn_info['Book_ID'] = matn_info['bookid_hadithid'].apply(lambda x: int(x.split('_')[0])) matn_info['Hadith Number'] = matn_info['bookid_hadithid'].apply(lambda x: int(x.split('_')[1])) matn_info = pd.merge(matn_info, books, on='Book_ID') def value_to_hex(value): rgba_color = cmap(value) return "#{:02X}{:02X}{:02X}".format(int(rgba_color[0] * 255), int(rgba_color[1] * 255), int(rgba_color[2] * 255)) def visualize_isnad(taraf_num, yaxis): taraf = matn_info[matn_info['taraf_ID'] == taraf_num] taraf_hadith = taraf['bookid_hadithid'].to_list() taraf_matns = taraf['matn'].to_list() taraf_hadith_split = [i.split('_') for i in taraf_hadith] taraf_book = taraf['Book_Name'].to_list() taraf_author = taraf['Author'].to_list() taraf_hadith_number = taraf['Hadith Number'].to_list() lst_hadith = [] hadith_cleaned = isnad_info['Tarafs Cleaned'].apply(lambda x: taraf_num in x) isnad_hadith = isnad_info[hadith_cleaned] for i in range(len(taraf_hadith_split)): # This checks each hadith in the Taraf, is that book id hadith id found in each of the edges of isnad_info #This loop get the end transmitter of each Hadith in the Taraf isnad_in_hadith1 = isnad_hadith['Hadiths Cleaned'].apply(lambda x: taraf_hadith_split[i] in x ) isnad_hadith1 = isnad_hadith[isnad_in_hadith1][['Source', 'Destination']] G = nx.from_pandas_edgelist(isnad_hadith1, source = 'Source', target = 'Destination', create_using = nx.DiGraph()) node = [int(n) for n, d in G.out_degree() if d == 0] for n in node: gen_node = narrator_bios[narrator_bios['Rawi ID']==n]['Generation'].to_list() if len(gen_node): gen_node = gen_node[0] else: gen_node = -1 name_node = narrator_bios[narrator_bios['Rawi ID']==n]['Famous Name'].to_list() if len(name_node): name_node = name_node[0] else: name_node = 'فلان' lst_hadith.append([taraf_matns[i], gen_node, name_node, taraf_book[i], taraf_author[i], taraf_hadith_number[i], str(n), str(i)]) df = pd.DataFrame(lst_hadith, columns = ['Matn', 'Generation', 'Name', 'Book_Name', 'Author', 'Book Hadith Number', 'End Transmitter ID', 'Hadith Number']) isnad_hadith['Teacher'] = isnad_hadith['Source'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list()) isnad_hadith['Student'] = isnad_hadith['Destination'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list()) isnad_hadith['Teacher'] = isnad_hadith['Teacher'].apply(lambda x: x[0] if len(x)==1 else 'فلان') isnad_hadith['Student'] = isnad_hadith['Student'].apply(lambda x: x[0] if len(x)==1 else 'فلان') end_nodes = df['End Transmitter ID'].tolist() G = nx.from_pandas_edgelist(isnad_hadith, source = 'Source', target = 'Destination', create_using = nx.DiGraph()) isnad_pos = nx.nx_agraph.graphviz_layout(G, prog='dot') x_stretch = 4 y_stretch = 4 net = Network(directed =True) for node, pos in isnad_pos.items(): node_info = narrator_bios[narrator_bios['Rawi ID'] == int(node)] student_narrations = node_info['Number of Narrations'].to_list() if len(student_narrations): student_narrations = student_narrations[0] else: student_narrations = 1 student_gen = node_info['Generation'].to_list() if len(student_gen): student_gen = student_gen[0] else: student_gen = -1 student_rank = node_info["Narrator Rank"].to_list() if len(student_rank): student_rank = student_rank[0] else: student_rank = 'فلان' node_name = node_info['Famous Name'].to_list() if len(node_name): node_name = node_name[0] else: node_name = 'فلان' if node == '99999': net.add_node(node, font = {'size':50, 'color': 'black'}, color = '#000000', label = f'{node_name} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 70) elif node in end_nodes: end_matn_info = df[df["End Transmitter ID"] == node] net.add_node(node, font = {'size':30, 'color': 'red'}, color = value_to_hex(student_narrations), label = f'{node_name} \n {student_rank} \n ID: {node} - Gen {student_gen} \n Hadith {" ".join(end_matn_info["Hadith Number"].tolist())}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 50) else: net.add_node(node, font = {'size':30, 'color': 'red'}, color = value_to_hex(student_narrations), label = f'{node_name} \n {student_rank} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 50) for _, row in isnad_hadith.iterrows(): source = row['Source'] target = row['Destination'] net.add_edge(source, target, color = value_to_hex(int(row[f'{yaxis} Count'])), label = f"{row[f'{yaxis} Count']}") net.toggle_physics(False) html = net.generate_html() html = html.replace("'", "\"") df = df.rename(columns = {'Generation': 'Gen.', 'Book Hadith Number': 'Hdth Num', 'End Transmitter ID': 'End Narrator ID', 'Hadith Number': 'Index', 'Book_Name': 'Book', 'Name':'Final Narrator'}) return f"""""" , df.drop('Hdth Num', axis=1) def visualize_subTaraf(taraf_num, hadith_str, yaxis): hadith_list = hadith_str.split(',') hadith_list = [hadith.strip() for hadith in hadith_list] hadiths = np.array([], dtype=int) for hadith in hadith_list: if '-' in hadith: if hadith.count('-') > 1: #print('Please use only one Dash mark!') raise gr.Error('Please use only one Dash mark!') hadith_multi = hadith.strip().split('-') if any([not had.isnumeric() for had in hadith_multi]): #print('Invalid Begining') raise gr.Error('Invalid Begining') elif len(hadith_multi) != 2: #print('Two numbers for a range of Hadith numbers please!') raise gr.Error('Two numbers for a range of Hadith numbers please!') hadith_multi = [int(had) for had in hadith_multi] hadiths = np.append(hadiths, np.arange(hadith_multi[0], hadith_multi[1])) elif hadith.isnumeric(): hadiths = np.append(hadiths, int(hadith)) else: #print('Invalid Data format!') raise gr.Error("Invalid Data format!") hadiths= np.unique(hadiths) taraf = matn_info[matn_info['taraf_ID'] == taraf_num] num_hadith = taraf.shape[0] if np.max(hadiths) > num_hadith: raise gr.Error(f'Hadith index outside of range. Total Number of Hadith in this Taraf: {num_hadith}') taraf['Index'] = np.arange(num_hadith) sub_taraf = taraf[taraf['Index'].isin(hadiths)] taraf_hadith = sub_taraf['bookid_hadithid'].to_list() taraf_hadith_split = [i.split('_') for i in taraf_hadith] hadith_cleaned = isnad_info['Tarafs Cleaned'].apply(lambda x: taraf_num in x) isnad_hadith = isnad_info[hadith_cleaned] subtaraf_filter = isnad_info['Hadiths Cleaned'].apply(lambda x: any(i in x for i in taraf_hadith_split)) isnad_hadith = isnad_hadith[subtaraf_filter] isnad_hadith['Teacher'] = isnad_hadith['Source'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list()) isnad_hadith['Student'] = isnad_hadith['Destination'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list()) isnad_hadith['Teacher'] = isnad_hadith['Teacher'].apply(lambda x: x[0] if len(x)==1 else 'فلان') isnad_hadith['Student'] = isnad_hadith['Student'].apply(lambda x: x[0] if len(x)==1 else 'فلان') G = nx.from_pandas_edgelist(isnad_hadith, source = 'Source', target = 'Destination', create_using = nx.DiGraph()) isnad_pos = nx.nx_agraph.graphviz_layout(G, prog='dot') x_stretch = 4 y_stretch = 4 net = Network(directed =True) for node, pos in isnad_pos.items(): node_info = narrator_bios[narrator_bios['Rawi ID'] == int(node)] student_narrations = node_info['Number of Narrations'].to_list() if len(student_narrations): student_narrations = student_narrations[0] else: student_narrations = 1 student_gen = node_info['Generation'].to_list() if len(student_gen): student_gen = student_gen[0] else: student_gen = -1 student_rank = node_info["Narrator Rank"].to_list() if len(student_rank): student_rank = student_rank[0] else: student_rank = 'فلان' node_name = node_info['Famous Name'].to_list() if len(node_name): node_name = node_name[0] else: node_name = 'فلان' if node == '99999': net.add_node(node, font = {'size':50, 'color': 'black'}, color = '#000000', label = f'{node_name} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 70) else: net.add_node(node, font = {'size':30, 'color': 'red'}, color = value_to_hex(student_narrations), label = f'{node_name} \n {student_rank} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 50) for _, row in isnad_hadith.iterrows(): source = row['Source'] target = row['Destination'] net.add_edge(source, target, color = value_to_hex(int(row[f'{yaxis} Count'])), label = f"{row[f'{yaxis} Count']}") net.toggle_physics(False) html = net.generate_html() html = html.replace("'", "\"") return f"""""", sub_taraf[['matn', 'Book_Name', 'Author', 'Book_ID', 'Hadith Number']] def taraf_booknum(taraf_num): taraf = matn_info[matn_info['taraf_ID'] == taraf_num] return taraf[['matn', 'Book_ID', 'Hadith Number', 'Book_Name', 'Author']] def visualize_hadith_isnad(df, yaxis): df['bookid_hadithid'] = df['Book_ID'].astype(str) + '_' + df['Hadith Number'].astype(str) hadith = matn_info[matn_info['bookid_hadithid'].isin(df['bookid_hadithid'])] taraf_hadith_split = [i.split('_') for i in hadith['bookid_hadithid'].to_list()] hadith_cleaned = isnad_info['Hadiths Cleaned'].apply(lambda x: any(i in x for i in taraf_hadith_split)) isnad_hadith = isnad_info[hadith_cleaned] isnad_hadith['Teacher'] = isnad_hadith['Source'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list()) isnad_hadith['Student'] = isnad_hadith['Destination'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list()) isnad_hadith['Teacher'] = isnad_hadith['Teacher'].apply(lambda x: x[0] if len(x)==1 else 'فلان') isnad_hadith['Student'] = isnad_hadith['Student'].apply(lambda x: x[0] if len(x)==1 else 'فلان') G = nx.from_pandas_edgelist(isnad_hadith, source = 'Source', target = 'Destination', create_using = nx.DiGraph()) isnad_pos = nx.nx_agraph.graphviz_layout(G, prog='dot') x_stretch = 4 y_stretch = 4 net = Network(directed =True) for node, pos in isnad_pos.items(): node_info = narrator_bios[narrator_bios['Rawi ID'] == int(node)] student_narrations = node_info['Number of Narrations'].to_list() if len(student_narrations): student_narrations = student_narrations[0] else: student_narrations = 1 student_gen = node_info['Generation'].to_list() if len(student_gen): student_gen = student_gen[0] else: student_gen = -1 student_rank = node_info["Narrator Rank"].to_list() if len(student_rank): student_rank = student_rank[0] else: student_rank = 'فلان' node_name = node_info['Famous Name'].to_list() if len(node_name): node_name = node_name[0] else: node_name = 'فلان' if node == '99999': net.add_node(node, font = {'size':50, 'color': 'black'}, color = '#000000', label = f'{node_name} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 70) else: net.add_node(node, font = {'size':30, 'color': 'red'}, color = value_to_hex(student_narrations), label = f'{node_name} \n {student_rank} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 50) for _, row in isnad_hadith.iterrows(): source = row['Source'] target = row['Destination'] net.add_edge(source, target, color = value_to_hex(int(row[f'{yaxis} Count'])), label = f"{row[f'{yaxis} Count']}") net.toggle_physics(False) html = net.generate_html() html = html.replace("'", "\"") return f"""""" def visualize_narrator_taraf(taraf_num, narrator, yaxis): taraf = matn_info[matn_info['taraf_ID'] == taraf_num] hadith_cleaned = isnad_info['Tarafs Cleaned'].apply(lambda x: taraf_num in x) isnad_hadith = isnad_info[hadith_cleaned] isnad_hadith['Teacher'] = isnad_hadith['Source'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list()) isnad_hadith['Student'] = isnad_hadith['Destination'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list()) isnad_hadith['Teacher'] = isnad_hadith['Teacher'].apply(lambda x: x[0] if len(x)==1 else 'فلان') isnad_hadith['Student'] = isnad_hadith['Student'].apply(lambda x: x[0] if len(x)==1 else 'فلان') taraf_hadith = taraf['bookid_hadithid'].to_list() taraf_hadith_split = [i.split('_') for i in taraf_hadith] G = nx.from_pandas_edgelist(isnad_hadith, source = 'Source', target = 'Destination', create_using = nx.DiGraph()) if narrator not in G.nodes(): raise gr.Error('Narrator not in Isnad of Taraf!') nodes = list(G.nodes) nodes = [node for node in nodes if node not in list(nx.ancestors(G, narrator)) + list(nx.descendants(G, narrator)) and node !=narrator] [G.remove_nodes_from(nodes)] isnad_pos = nx.nx_agraph.graphviz_layout(G, prog='dot') matns_with_narrator = [] for i in range(len(taraf_hadith_split)): isnad_in_hadith1 = isnad_hadith['Hadiths Cleaned'].apply(lambda x: taraf_hadith_split[i] in x ) isnad_hadith1 = isnad_hadith[isnad_in_hadith1] G1 = nx.from_pandas_edgelist(isnad_hadith1, source = 'Source', target = 'Destination', create_using = nx.DiGraph()) if narrator in list(G1.nodes): matns_with_narrator.append(taraf_hadith[i]) narrator_matn_info = taraf[taraf['bookid_hadithid'].isin(matns_with_narrator)] x_stretch = 4 y_stretch = 4 net = Network(directed =True) for node, pos in isnad_pos.items(): node_info = narrator_bios[narrator_bios['Rawi ID'] == int(node)] student_narrations = node_info['Number of Narrations'].to_list() if len(student_narrations): student_narrations = student_narrations[0] else: student_narrations = 1 student_gen = node_info['Generation'].to_list() if len(student_gen): student_gen = student_gen[0] else: student_gen = -1 student_rank = node_info["Narrator Rank"].to_list() if len(student_rank): student_rank = student_rank[0] else: student_rank = 'فلان' node_name = node_info['Famous Name'].to_list() if len(node_name): node_name = node_name[0] else: node_name = 'فلان' if node == '99999': net.add_node(node, font = {'size':50, 'color': 'black'}, color = '#000000', label = f'{node_name} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 70) else: net.add_node(node, font = {'size':30, 'color': 'red'}, color = value_to_hex(student_narrations), label = f'{node_name} \n {student_rank} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 50) for edge in G.edges: row = isnad_hadith[(isnad_hadith['Source'] == edge[0]) & (isnad_hadith['Destination'] == edge[1])].iloc[0] source = row['Source'] target = row['Destination'] net.add_edge(source, target, color = value_to_hex(int(row[f'{yaxis} Count'])), label = f"{row[f'{yaxis} Count']}") net.toggle_physics(False) html = net.generate_html() html = html.replace("'", "\"") return f"""""" , narrator_matn_info[['matn', 'Book_Name', 'Author', 'Book_ID', 'Hadith Number']] with gr.Blocks() as demo: with gr.Tab("Whole Taraf Visualizer"): Yaxis = gr.Dropdown(choices = ['Taraf', 'Hadith', 'Isnad', 'Book'], value = 'Taraf', label = 'Variable to Display', info = 'Choose the variable to visualize.') taraf_number = gr.Slider(1,taraf_max , value=10000, label="Taraf", info="Choose the Taraf to Input", step = 1) btn = gr.Button('Submit') btn.click(fn = visualize_isnad, inputs = [taraf_number, Yaxis], outputs = [gr.HTML(), gr.DataFrame(wrap=True, column_widths=[43, 8, 11,11,10,8, 9])]) with gr.Tab("Book and Hadith Number Retriever"): taraf_num = gr.Slider(1,taraf_max , value=10000, label="Taraf", info="Choose the Taraf to Input", step = 1) btn_num = gr.Button('Retrieve') btn_num.click(fn=taraf_booknum, inputs = [taraf_num], outputs= [gr.DataFrame(wrap=True)]) with gr.Tab('Sub Taraf Visualizer'): taraf_num = gr.Slider(1,taraf_max , value=10000, label="Taraf", info="Choose the Taraf to Input", step = 1) Yaxis = gr.Dropdown(choices = ['Taraf', 'Hadith', 'Isnad', 'Book'], value = 'Taraf', label = 'Variable to Display', info = 'Choose the variable to visualize.') hadith_str = gr.Textbox(label='Hadith Selection', info='Choose which range of Hadith you would like visualized from the Taraf (eg "1, 2, 4-7")') btn_sub = gr.Button('Visualize') btn_sub.click(fn=visualize_subTaraf, inputs = [taraf_num, hadith_str, Yaxis], outputs=[gr.HTML(), gr.DataFrame(wrap=True)]) with gr.Tab('Select Hadith Isnad Visualizer'): yyaxis = gr.Dropdown(choices = ['Taraf', 'Hadith', 'Isnad', 'Book'], value = 'Taraf', label = 'Variable to Display', info = 'Choose the variable to visualize.') hadith_selection = gr.Dataframe( headers=["Book_ID", "Hadith Number"], datatype=["number", "number"], row_count=5, col_count=(2, "fixed")) btn_hadith = gr.Button('Visualize') btn_hadith.click(fn=visualize_hadith_isnad, inputs=[hadith_selection, yyaxis], outputs=[gr.HTML()]) with gr.Tab('Taraf Narrator Isnad Visualizer'): Yaxis = gr.Dropdown(choices = ['Taraf', 'Hadith', 'Isnad', 'Book'], value = 'Taraf', label = 'Variable to Display', info = 'Choose the variable to visualize.') taraf_number = gr.Slider(1,taraf_max , value=10000, label="Taraf", info="Choose the Taraf to Input", step = 1) narr = gr.Textbox(label='Narrator', info='Choose a Narrator (Refer to full isnad from previous tab)') btn_narr = gr.Button('Visualize') btn_narr.click(fn=visualize_narrator_taraf, inputs=[taraf_number, narr, Yaxis], outputs=[gr.HTML(), gr.DataFrame(wrap=True)]) demo.launch()