Upload 2 files

dashboard.py

@@ -0,0 +1,424 @@
+
+import warnings
+warnings.filterwarnings("ignore")
+import io
+import os
+import time
+import warnings
+warnings.simplefilter(action='ignore', category=FutureWarning)
+warnings.simplefilter(action='ignore', category=RuntimeWarning)
+import pandas as pd
+import csv
+import ast
+from tqdm import tqdm
+from operator import itemgetter
+import numpy as np
+import re
+import datetime
+import html
+from joblib import Parallel, delayed
+import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+#plt.style.use('seaborn-paper')
+import holoviews as hv
+from holoviews import opts, dim
+from bokeh.sampledata.les_mis import data
+from bokeh.io import show
+from bokeh.sampledata.les_mis import data
+import panel as pn
+import bokeh
+from bokeh.resources import INLINE
+from holoviews.operation.timeseries import rolling, rolling_outlier_std
+hv.extension('bokeh')
+
+## LOAD DATASETS
+
+dna_folder = './data'
+
+
+#### full data unfiltered:
+
+dna_articles_unfiltered_eu_time_indexed_resampled = pd.read_csv(os.path.join(dna_folder, 'dna_articles_unfiltered_eu_time_indexed_resampled.tsv'),sep='\t',header=0)
+dna_articles_unfiltered_us_time_indexed_resampled = pd.read_csv(os.path.join(dna_folder, 'dna_articles_unfiltered_us_time_indexed_resampled.tsv'),sep='\t',header=0)
+dna_articles_unfiltered_eu_us_time_indexed_resampled = pd.read_csv(os.path.join(dna_folder, 'dna_articles_unfiltered_eu_us_time_indexed_resampled.tsv'),sep='\t',header=0)
+
+#### classifier filtered articles:
+
+dh_ration_df_eu = pd.read_csv(os.path.join(dna_folder, 'dh_ration_df_eu.tsv'),sep='\t',header=0)
+dh_ration_df_us = pd.read_csv(os.path.join(dna_folder, 'dh_ration_df_us.tsv'),sep='\t',header=0)
+dh_ration_df_eu_us = pd.read_csv(os.path.join(dna_folder, 'dh_ration_df_eu_us.tsv'),sep='\t',header=0)
+
+regions = ['eu', 'us', 'eu_us']
+
+sorted_ent_type_freq_map_eu=dict()
+sorted_ent_type_freq_map_us=dict()
+sorted_ent_type_freq_map_eu_us=dict()
+
+def read_top_ent_types():
+    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_type_freq_map_eu.tsv'), 'r'))
+    for i,row in enumerate(reader):
+        if i < 20:
+            k, v = row
+            sorted_ent_type_freq_map_eu[k] = int(v)
+    del sorted_ent_type_freq_map_eu['Entity']
+    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_type_freq_map_us.tsv'), 'r'))
+    for i, row in enumerate(reader):
+        if i < 20:
+            k, v = row
+            sorted_ent_type_freq_map_us[k] = int(v)
+    del sorted_ent_type_freq_map_us['Entity']
+    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_type_freq_map_eu_us.tsv'), 'r'))
+    for i, row in enumerate(reader):
+        if i < 20:
+            k, v = row
+            sorted_ent_type_freq_map_eu_us[k] = int(v)
+    del sorted_ent_type_freq_map_eu_us['Entity']
+
+read_top_ent_types()
+
+top_type_filtered_eu = ['DBpedia:Country', 'DBpedia:Organisation', 'DBpedia:Company', 'DBpedia:Person', 'DBpedia:Disease', 'DBpedia:ChemicalSubstance', 'DBpedia:Drug', 'DBpedia:GovernmentAgency', 'DBpedia:City', 'DBpedia:MonoclonalAntibody']
+top_type_filtered_us = ['DBpedia:Organisation', 'DBpedia:Company', 'DBpedia:Disease', 'DBpedia:ChemicalSubstance', 'DBpedia:Person', 'DBpedia:Drug', 'DBpedia:Country', 'DBpedia:Region', 'DBpedia:MonoclonalAntibody', 'DBpedia:City', 'DBpedia:Biomolecule']
+top_type_filtered_eu_us =  ['DBpedia:Organisation', 'DBpedia:Company', 'DBpedia:ChemicalSubstance', 'DBpedia:Drug', 'DBpedia:Country', 'DBpedia:Person', 'DBpedia:Disease', 'DBpedia:MonoclonalAntibody', 'DBpedia:GovernmentAgency', 'DBpedia:Biomolecule', 'DBpedia:Gene']
+
+dna_healthtech_articles_eu_time_indexed_resampled=pd.read_csv(os.path.join(dna_folder, 'dna_healthtech_articles_eu_time_indexed_resampled.tsv'),sep='\t',header=0)
+dna_healthtech_articles_us_time_indexed_resampled=pd.read_csv(os.path.join(dna_folder, 'dna_healthtech_articles_us_time_indexed_resampled.tsv'),sep='\t',header=0)
+dna_healthtech_articles_eu_us_time_indexed_resampled=pd.read_csv(os.path.join(dna_folder, 'dna_healthtech_articles_eu_us_time_indexed_resampled.tsv'),sep='\t',header=0)
+
+def read_top_ent_maps():
+    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_freq_map_eu.tsv'), 'r'), delimiter='\t')
+    for row in reader:
+        k,v = row
+        lista = ast.literal_eval(v)
+        dizionario = dict()
+        for pair in lista:
+            dizionario[pair[0]]=pair[1]
+        dizionario = sorted(dizionario.items(), key=lambda x: x[1], reverse=True)
+        ent_freq_maps_eu[k]=dizionario
+
+    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_freq_map_us.tsv'), 'r'), delimiter='\t')
+    for row in reader:
+        k, v = row
+        lista = ast.literal_eval(v)
+        dizionario = dict()
+        for pair in lista:
+            dizionario[pair[0]] = pair[1]
+        dizionario = sorted(dizionario.items(), key=lambda x: x[1], reverse=True)
+        ent_freq_maps_us[k] = dizionario
+
+    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_freq_map_eu_us.tsv'), 'r'), delimiter='\t')
+    for row in reader:
+        k, v = row
+        lista = ast.literal_eval(v)
+        dizionario = dict()
+        for pair in lista:
+            dizionario[pair[0]] = pair[1]
+        dizionario = sorted(dizionario.items(), key=lambda x: x[1], reverse=True)
+        ent_freq_maps_eu_us[k] = dizionario
+
+ent_freq_maps_eu = dict()
+ent_freq_maps_us = dict()
+ent_freq_maps_eu_us = dict()
+
+read_top_ent_maps()
+
+
+def read_type_filtered_triples():
+    for t in top_type_filtered_eu:
+        df = pd.read_csv(dna_folder+'/filtered_rows/eu/'+t.replace(':','_')+'.tsv', sep="	", header=0)
+        df.drop(columns=['Unnamed: 0'], inplace=True)
+        top_type_filtered_triples_eu[t]=df
+    for t in top_type_filtered_us:
+        df = pd.read_csv(dna_folder+'/filtered_rows/us/'+t.replace(':','_')+'.tsv', sep="	")
+        df.drop(columns=['Unnamed: 0'], inplace=True)
+        top_type_filtered_triples_us[t]=df
+    for t in top_type_filtered_eu_us:
+        df = pd.read_csv(dna_folder+'/filtered_rows/eu_us/'+t.replace(':','_')+'.tsv', sep="	")
+        df.drop(columns=['Unnamed: 0'], inplace=True)
+        top_type_filtered_triples_eu_us[t]=df
+
+
+
+top_type_filtered_triples_eu = dict()
+top_type_filtered_triples_us = dict()
+top_type_filtered_triples_eu_us = dict()
+
+read_type_filtered_triples()
+
+grouping_filtered = pd.read_csv(os.path.join(dna_folder, 'dna_relations.tsv'), sep="	")
+################################# CREATE CHARTS ############################
+def create_curve_chart():
+    # Create the 3 line plots
+    curve_eu = hv.Curve((dh_ration_df_eu.index, dh_ration_df_eu.ids/dna_articles_unfiltered_eu_time_indexed_resampled.ids), 'Time', 'Digital Health News Ratio',label='EU')
+    curve_us = hv.Curve((dh_ration_df_us.index, dh_ration_df_us.ids/dna_articles_unfiltered_us_time_indexed_resampled.ids),'Time', 'Digital Health News Ratio', label='US')
+    curve_eu_us = hv.Curve((dh_ration_df_eu_us.index, dh_ration_df_eu_us.ids/dna_articles_unfiltered_eu_us_time_indexed_resampled.ids),'Time', 'Digital Health News Ratio', label='EU-US')
+    #Overlay the line plots
+    overlay = curve_eu * curve_us * curve_eu_us
+    overlay.opts(show_legend = True, legend_position='top_left', width=1200, height=600)
+    return overlay
+
+
+def create_bar_charts(region, **kwargs):
+  if region=='eu':
+    sliced = sorted_ent_type_freq_map_eu
+    return hv.Bars(sliced, hv.Dimension('Entity Types'), 'Frequency').opts( framewise=True, xrotation=45,width=1200, height=600)
+  elif region=='us':
+    sliced = sorted_ent_type_freq_map_us
+    return hv.Bars(sliced, hv.Dimension('Entity Types'), 'Frequency').opts(framewise=True, xrotation=45,width=1200, height=600)
+  elif region=='eu_us':
+    sliced = sorted_ent_type_freq_map_eu_us
+    return hv.Bars(sliced, hv.Dimension('Entity Types'), 'Frequency').opts(framewise=True, xrotation=45,width=1200, height=600)
+
+
+
+# Define a function to generate Curve based on selected values
+def generate_entity_curves(region_value, type_value, **kwargs):
+  if region_value=='eu':
+    top20Ents = ent_freq_maps_eu[type_value]
+    curveList = []
+    for ent in top20Ents:
+      entityTriples =  top_type_filtered_triples_eu[type_value][(top_type_filtered_triples_eu[type_value]['subjEntityLinks']==ent[0]) | (top_type_filtered_triples_eu[type_value]['objEntityLinks']==ent[0])]
+      entityTriples_time_indexed = entityTriples.set_index(pd.DatetimeIndex(entityTriples['timestamp']), inplace=False)
+      del entityTriples_time_indexed['timestamp']
+      entityTriples_time_indexed_resampled = entityTriples_time_indexed.resample("Y").count()
+      #print(entityTriples_time_indexed_resampled)
+      entityTriples_time_indexed_resampled = entityTriples_time_indexed_resampled.reindex(dna_healthtech_articles_eu_time_indexed_resampled.index, fill_value=0)
+      curve = hv.Curve((entityTriples_time_indexed_resampled.index, (entityTriples_time_indexed_resampled['doc_id']/dna_healthtech_articles_eu_time_indexed_resampled['ids'])), 'Time', 'Key Entity Occurrence', label=ent[0])
+      curve.opts(autorange='y')
+      #curve.opts(logy=True)
+      curveList.append(curve)
+    overlay = hv.Overlay(curveList)
+    overlay.opts(legend_muted=False, legend_cols=4, show_legend = True, legend_position='top_left', fontsize={'legend':13},width=1200, height=800)
+    return overlay
+
+  elif region_value=='us':
+    top20Ents = ent_freq_maps_us[type_value]
+    curveList = []
+    for ent in top20Ents:
+      entityTriples =  top_type_filtered_triples_us[type_value][(top_type_filtered_triples_us[type_value]['subjEntityLinks']==ent[0]) | (top_type_filtered_triples_us[type_value]['objEntityLinks']==ent[0])]
+      entityTriples_time_indexed = entityTriples.set_index(pd.DatetimeIndex(entityTriples['timestamp']), inplace=False)
+      del entityTriples_time_indexed['timestamp']
+      entityTriples_time_indexed_resampled = entityTriples_time_indexed_resampled.reindex(dna_healthtech_articles_us_time_indexed_resampled.index, fill_value=0)
+      curve = hv.Curve((entityTriples_time_indexed_resampled.index, (entityTriples_time_indexed_resampled['doc_id']/dna_healthtech_articles_us_time_indexed_resampled['ids'])), 'Time', 'Key Entity Occurrence', label=ent[0])
+      curve.opts(autorange='y')
+      curveList.append(curve)
+    overlay = hv.Overlay(curveList)
+    overlay.opts(legend_muted=False, legend_cols=4, show_legend = True, legend_position='top_left', fontsize={'legend':13},width=1200, height=800)
+    return overlay
+
+  elif region_value=='eu_us':
+    top20Ents = ent_freq_maps_eu_us[type_value]
+    curveList = []
+    for ent in top20Ents:
+      entityTriples =  top_type_filtered_triples_eu_us[type_value][(top_type_filtered_triples_eu_us[type_value]['subjEntityLinks']==ent[0]) | (top_type_filtered_triples_eu_us[type_value]['objEntityLinks']==ent[0])]
+      entityTriples_time_indexed = entityTriples.set_index(pd.DatetimeIndex(entityTriples['timestamp']), inplace=False)
+      del entityTriples_time_indexed['timestamp']
+      entityTriples_time_indexed_resampled = entityTriples_time_indexed_resampled.reindex(dna_healthtech_articles_eu_us_time_indexed_resampled.index, fill_value=0)
+      curve = hv.Curve((entityTriples_time_indexed_resampled.index, (entityTriples_time_indexed_resampled['doc_id']/dna_healthtech_articles_eu_us_time_indexed_resampled['ids'])), 'Time', 'Key Entity Occurrence', label=ent[0])
+      curve.opts(autorange='y')
+      curveList.append(curve)
+    overlay = hv.Overlay(curveList)
+    overlay.opts(legend_muted=False, legend_cols=4, show_legend = True, legend_position='top_left', fontsize={'legend':13},width=1200, height=800)
+    return overlay
+
+
+############################# WIDGETS & CALLBACK ###########################################
+
+def filter_data0(df, min_value):
+    filtered_df = df[df['value'] >= min_value]
+    return filtered_df
+
+
+def plot_chord0_new(df,min_value):
+    filtered_df = filter_data0(df, min_value)
+  # Create a Holoviews Dataset for nodes
+    nodes = hv.Dataset(filtered_df, 'index')
+    nodes.data.head()
+    chord = hv.Chord(filtered_df, ['source', 'target'], ['value'])
+    return chord.opts(opts.Chord(cmap='Category20', edge_cmap='Category20', label_text_color="white",  node_color = hv.dim('index').str(),  edge_color = hv.dim('source').str(), labels = 'index', tools=['hover'],   width=800, height=800))
+
+
+def retrieveRegionTypes(region):
+  if region == 'eu':
+    return top_type_filtered_eu
+  elif region == 'us':
+    return top_type_filtered_us
+  elif region == 'eu_us':
+    return top_type_filtered_eu_us
+
+
+def filter_region(region):
+    if region == 'eu':
+        region_grouping = grouping_filtered[grouping_filtered['region'] == 'eu']
+    elif region == 'us':
+        region_grouping = grouping_filtered[grouping_filtered['region'] == 'us']
+    elif region == 'eu_us':
+        region_grouping = grouping_filtered[grouping_filtered['region'] == 'eu_us']
+
+    #print(len(region_grouping))
+    # Define range for minimum value slider
+    min_value_range = region_grouping['value'].unique()
+    min_value_range.sort()
+
+    # Define HoloMap with minimum value and attribute as key dimensions
+    holomap = hv.HoloMap({min_value: plot_chord0_new(region_grouping, min_value)
+                          for min_value in min_value_range},
+                         kdims=['Show triples with support greater than']
+                         )
+    return holomap
+
+
+# Define a function to generate Entity List RadioButtonGroup based on Region selection
+def generate_radio_buttons(value):
+    if value == 'eu':
+        return pn.widgets.RadioButtonGroup(options=retrieveRegionTypes(value), value='DBpedia:Company', name='eu', orientation='vertical')
+    elif value == 'us':
+        return pn.widgets.RadioButtonGroup(options=retrieveRegionTypes(value), value='DBpedia:Disease', name='us', orientation='vertical')
+    elif value == 'eu_us':
+        return pn.widgets.RadioButtonGroup(options=retrieveRegionTypes(value), value='DBpedia:Person', name='eu_us', orientation='vertical')
+
+
+
+# https://tabler-icons.io/
+button1 = pn.widgets.Button(name="Introduction", button_type="warning", icon="file-info", styles={"width": "100%"})
+button2 = pn.widgets.Button(name="Health Tech News Ratio", button_type="warning",  icon="chart-histogram", styles={"width": "100%"})
+button3 = pn.widgets.Button(name="Top Entity Types", button_type="warning", icon="chart-bar", styles={"width": "100%"})
+button4 = pn.widgets.Button(name="Top Key Entities", button_type="warning", icon="chart-dots-filled", styles={"width": "100%"})
+button5 = pn.widgets.Button(name="Entity Chord Diagrams", button_type="warning", icon="chart-dots-filled", styles={"width": "100%"})
+
+
+region1 = pn.widgets.RadioButtonGroup(name='### Select News Region', options=regions)
+
+
+# Initial RadioButtonGroup
+radio_buttons_regions =  pn.widgets.RadioButtonGroup(options=regions,value='eu',name='Select region')
+# Generate initial dynamic RadioButtonGroup
+radio_buttons_types  = generate_radio_buttons(radio_buttons_regions.value)
+
+
+
+# Define a callback function to update the panel dynamically
+def update_radio_group(event):
+  #print(event.new)
+  #print(retrieveRegionTypes(event.new))
+  radio_buttons_types.options = retrieveRegionTypes(event.new)
+
+
+# bind the function to the widget(s)
+dmap2 = hv.DynamicMap(pn.bind(generate_entity_curves, radio_buttons_regions,radio_buttons_types))
+# Bind the selected value of the first RadioButtonGroup to update the second RadioButtonGroup
+radio_buttons_regions.param.watch(update_radio_group, 'value')
+
+# Define the callback function to update the HoloMap
+def update_holomap(event):
+    initial_holomap.object = filter_region(event.new)
+
+region_radio_button = pn.widgets.RadioButtonGroup(options=regions, value='eu', name='Select Region')
+
+# Create the initial HoloMap
+initial_holomap = filter_region(region_radio_button.value)
+
+# Bind the callback function to the value change event of the RadioButton widget
+region_radio_button.param.watch(update_holomap, 'value')
+
+
+
+def show_page(page_key):
+    main_area.clear()
+    main_area.append(mapping[page_key])
+
+button1.on_click(lambda event: show_page("Page1"))
+button2.on_click(lambda event: show_page("Page2"))
+button3.on_click(lambda event: show_page("Page3"))
+button4.on_click(lambda event: show_page("Page4"))
+button5.on_click(lambda event: show_page("Page5"))
+
+
+### CREATE PAGE LAYOUTS
+
+def CreatePage1():
+    return pn.Column(pn.pane.Markdown("""
+
+This is a dashboard for a News Analysis project regarding Digital Health technology. The source data consists of around 7.8 million English-language news articles gathered from the **Dow Jones Data, News, and Analytics (DNA)**
+platform (https://www.dowjones.com/professional/developer-platform/) covering a timeframe from September 1987 through December 2023. The news items text content is copyrighted and cannot be shared within this project.
+
+Some of the data analytics visualizations show here come from a Knowledge Graph automatically extracted from DNA news sources. A Virtuoso SPARQL endpoint to this graph (named 'DHNEWS KG') is set up at the 
+URL: https://api-vast.jrc.service.ec.europa.eu/sparql/
+
+
+---------------------------
+
+## 1. Health Tech News Ratio 
+In the Health Tech News Ratio panel we present the month-sampled time series depicting the proportion of 97k news articles con-
+cerning Digital Health, as identified by a text classifier, out of the total number of English language DNA news articles pertaining to Europe and the US
+
+
+### 2. Top Entity Types 
+The Top Entity Types bar plots in the dashboard show the predominant DBpedia-inherited entity types within the graph for triples tagged with Europe, US, and EU-US region codes via their article support.
+
+## 3. Top Key Entities 
+The Top Key Entities plots track the occurrence of several key entities per year, where occurrence means the entity is either the Subject or Object of an extracted triple in the KG.
+
+## 4. Entity Chord Diagrams  
+Entity Chord Diagrams represent the most frequently connected entity pairs within the KG through chord illustrations, serving as both Subjects and Objects of predicative triples. 
+The size of the chords corresponds to the support of the depicted relations.
+""", width=800), align="center")
+
+def CreatePage2():
+    return pn.Column(
+        pn.pane.Markdown("## Health Tech News Ratio "),
+        create_curve_chart(),
+        align="center",
+    )
+
+def CreatePage3():
+    return pn.Column(
+    region1,
+        pn.bind(create_bar_charts, region1),
+        align="center",
+    )
+
+def CreatePage4():
+    return pn.Column(
+        pn.pane.Markdown("## Top Key Entities "),
+        pn.Row(pn.Column(radio_buttons_regions, radio_buttons_types), dmap2),
+        align="center", )
+
+def CreatePage5():
+    return pn.Column(
+        pn.pane.Markdown("## Entity Chord Diagrams "),
+        pn.Row(region_radio_button, pn.bind(filter_region, region_radio_button)),
+        align="center", )
+
+mapping = {
+    "Page1": CreatePage1(),
+    "Page2": CreatePage2(),
+    "Page3": CreatePage3(),
+    "Page4": CreatePage4(),
+    "Page5": CreatePage5(),
+}
+
+#################### SIDEBAR LAYOUT ##########################
+sidebar = pn.Column(pn.pane.Markdown("## Pages"), button1,button2,button3,
+                    button4,
+                    button5,
+					styles={"width": "100%", "padding": "15px"})
+
+#################### MAIN AREA LAYOUT ##########################
+main_area = pn.Column(mapping["Page1"], styles={"width":"100%"})
+
+###################### APP LAYOUT ##############################
+template = pn.template.BootstrapTemplate(
+    title=" Digital Health in the News: Analytics Dashboard ",
+    sidebar=[sidebar],
+    main=[main_area],
+    header_background="black",
+    #site="Charting the Landscape of Digital Health",
+    theme=pn.template.DarkTheme,
+    sidebar_width=250, ## Default is 330
+    busy_indicator=pn.indicators.BooleanStatus(value=True),
+)
+
+### DEPLOY APP
+
+# Serve the Panel app
+template.servable()

requirements.txt

@@ -1,3 +1,37 @@
-panel
-pandas
-hvplot
+bleach==4.1.0
+bokeh==2.3.3
+certifi==2024.2.2
+charset-normalizer==2.0.12
+colorama==0.4.5
+colorcet==3.0.1
+cycler==0.11.0
+holoviews==1.14.9
+idna==3.7
+importlib-metadata==4.8.3
+importlib-resources==5.4.0
+Jinja2==3.0.3
+joblib==1.1.1
+kiwisolver==1.3.1
+Markdown==3.3.7
+MarkupSafe==2.0.1
+matplotlib==3.3.4
+numpy==1.19.5
+packaging==21.3
+pandas==1.1.5
+panel==0.12.1
+param==1.13.0
+Pillow==8.4.0
+pyct==0.4.8
+pyparsing==3.0.7
+python-dateutil==2.9.0.post0
+pytz==2024.1
+pyviz-comms==2.2.1
+PyYAML==6.0.1
+requests==2.27.1
+six==1.16.0
+tornado==6.1
+tqdm==4.64.1
+typing_extensions==4.1.1
+urllib3==1.26.18
+webencodings==0.5.1
+zipp==3.6.0