dashboard.py

import warnings
warnings.filterwarnings("ignore")
import io
import os
import time
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
warnings.simplefilter(action='ignore', category=RuntimeWarning)
import pandas as pd
import csv
import ast
from tqdm import tqdm
from operator import itemgetter
import numpy as np
import re
import datetime
import html
from joblib import Parallel, delayed
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
#plt.style.use('seaborn-paper')
import holoviews as hv
from holoviews import opts, dim
from bokeh.sampledata.les_mis import data
from bokeh.io import show
from bokeh.sampledata.les_mis import data
import panel as pn
import bokeh
from bokeh.resources import INLINE
from holoviews.operation.timeseries import rolling, rolling_outlier_std
hv.extension('bokeh')

## LOAD DATASETS

dna_folder = './data'


#### full data unfiltered:

dna_articles_unfiltered_eu_time_indexed_resampled = pd.read_csv(os.path.join(dna_folder, 'dna_articles_unfiltered_eu_time_indexed_resampled.tsv'),sep='\t',header=0)
dna_articles_unfiltered_us_time_indexed_resampled = pd.read_csv(os.path.join(dna_folder, 'dna_articles_unfiltered_us_time_indexed_resampled.tsv'),sep='\t',header=0)
dna_articles_unfiltered_eu_us_time_indexed_resampled = pd.read_csv(os.path.join(dna_folder, 'dna_articles_unfiltered_eu_us_time_indexed_resampled.tsv'),sep='\t',header=0)

#### classifier filtered articles:

dh_ration_df_eu = pd.read_csv(os.path.join(dna_folder, 'dh_ration_df_eu.tsv'),sep='\t',header=0)
dh_ration_df_us = pd.read_csv(os.path.join(dna_folder, 'dh_ration_df_us.tsv'),sep='\t',header=0)
dh_ration_df_eu_us = pd.read_csv(os.path.join(dna_folder, 'dh_ration_df_eu_us.tsv'),sep='\t',header=0)

regions = ['eu', 'us', 'eu_us']

sorted_ent_type_freq_map_eu=dict()
sorted_ent_type_freq_map_us=dict()
sorted_ent_type_freq_map_eu_us=dict()

def read_top_ent_types():
    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_type_freq_map_eu.tsv'), 'r'))
    for i,row in enumerate(reader):
        if i < 20:
            k, v = row
            sorted_ent_type_freq_map_eu[k] = int(v)
    del sorted_ent_type_freq_map_eu['Entity']
    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_type_freq_map_us.tsv'), 'r'))
    for i, row in enumerate(reader):
        if i < 20:
            k, v = row
            sorted_ent_type_freq_map_us[k] = int(v)
    del sorted_ent_type_freq_map_us['Entity']
    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_type_freq_map_eu_us.tsv'), 'r'))
    for i, row in enumerate(reader):
        if i < 20:
            k, v = row
            sorted_ent_type_freq_map_eu_us[k] = int(v)
    del sorted_ent_type_freq_map_eu_us['Entity']

read_top_ent_types()

top_type_filtered_eu = ['DBpedia:Country', 'DBpedia:Organisation', 'DBpedia:Company', 'DBpedia:Person', 'DBpedia:Disease', 'DBpedia:ChemicalSubstance', 'DBpedia:Drug', 'DBpedia:GovernmentAgency', 'DBpedia:City', 'DBpedia:MonoclonalAntibody']
top_type_filtered_us = ['DBpedia:Organisation', 'DBpedia:Company', 'DBpedia:Disease', 'DBpedia:ChemicalSubstance', 'DBpedia:Person', 'DBpedia:Drug', 'DBpedia:Country', 'DBpedia:Region', 'DBpedia:MonoclonalAntibody', 'DBpedia:City', 'DBpedia:Biomolecule']
top_type_filtered_eu_us =  ['DBpedia:Organisation', 'DBpedia:Company', 'DBpedia:ChemicalSubstance', 'DBpedia:Drug', 'DBpedia:Country', 'DBpedia:Person', 'DBpedia:Disease', 'DBpedia:MonoclonalAntibody', 'DBpedia:GovernmentAgency', 'DBpedia:Biomolecule', 'DBpedia:Gene']

dna_healthtech_articles_eu_time_indexed_resampled=pd.read_csv(os.path.join(dna_folder, 'dna_healthtech_articles_eu_time_indexed_resampled.tsv'),sep='\t',header=0)
dna_healthtech_articles_us_time_indexed_resampled=pd.read_csv(os.path.join(dna_folder, 'dna_healthtech_articles_us_time_indexed_resampled.tsv'),sep='\t',header=0)
dna_healthtech_articles_eu_us_time_indexed_resampled=pd.read_csv(os.path.join(dna_folder, 'dna_healthtech_articles_eu_us_time_indexed_resampled.tsv'),sep='\t',header=0)

def read_top_ent_maps():
    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_freq_map_eu.tsv'), 'r'), delimiter='\t')
    for row in reader:
        k,v = row
        lista = ast.literal_eval(v)
        dizionario = dict()
        for pair in lista:
            dizionario[pair[0]]=pair[1]
        dizionario = sorted(dizionario.items(), key=lambda x: x[1], reverse=True)
        ent_freq_maps_eu[k]=dizionario

    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_freq_map_us.tsv'), 'r'), delimiter='\t')
    for row in reader:
        k, v = row
        lista = ast.literal_eval(v)
        dizionario = dict()
        for pair in lista:
            dizionario[pair[0]] = pair[1]
        dizionario = sorted(dizionario.items(), key=lambda x: x[1], reverse=True)
        ent_freq_maps_us[k] = dizionario

    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_freq_map_eu_us.tsv'), 'r'), delimiter='\t')
    for row in reader:
        k, v = row
        lista = ast.literal_eval(v)
        dizionario = dict()
        for pair in lista:
            dizionario[pair[0]] = pair[1]
        dizionario = sorted(dizionario.items(), key=lambda x: x[1], reverse=True)
        ent_freq_maps_eu_us[k] = dizionario

ent_freq_maps_eu = dict()
ent_freq_maps_us = dict()
ent_freq_maps_eu_us = dict()

read_top_ent_maps()


def read_type_filtered_triples():
    for t in top_type_filtered_eu:
        df = pd.read_csv(dna_folder+'/filtered_rows/eu/'+t.replace(':','_')+'.tsv', sep="	", header=0)
        df.drop(columns=['Unnamed: 0'], inplace=True)
        top_type_filtered_triples_eu[t]=df
    for t in top_type_filtered_us:
        df = pd.read_csv(dna_folder+'/filtered_rows/us/'+t.replace(':','_')+'.tsv', sep="	")
        df.drop(columns=['Unnamed: 0'], inplace=True)
        top_type_filtered_triples_us[t]=df
    for t in top_type_filtered_eu_us:
        df = pd.read_csv(dna_folder+'/filtered_rows/eu_us/'+t.replace(':','_')+'.tsv', sep="	")
        df.drop(columns=['Unnamed: 0'], inplace=True)
        top_type_filtered_triples_eu_us[t]=df



top_type_filtered_triples_eu = dict()
top_type_filtered_triples_us = dict()
top_type_filtered_triples_eu_us = dict()

read_type_filtered_triples()

grouping_filtered = pd.read_csv(os.path.join(dna_folder, 'dna_relations.tsv'), sep="	")
################################# CREATE CHARTS ############################
def create_curve_chart():
    # Create the 3 line plots
    curve_eu = hv.Curve((dh_ration_df_eu.index, dh_ration_df_eu.ids/dna_articles_unfiltered_eu_time_indexed_resampled.ids), 'Time', 'Digital Health News Ratio',label='EU')
    curve_us = hv.Curve((dh_ration_df_us.index, dh_ration_df_us.ids/dna_articles_unfiltered_us_time_indexed_resampled.ids),'Time', 'Digital Health News Ratio', label='US')
    curve_eu_us = hv.Curve((dh_ration_df_eu_us.index, dh_ration_df_eu_us.ids/dna_articles_unfiltered_eu_us_time_indexed_resampled.ids),'Time', 'Digital Health News Ratio', label='EU-US')
    #Overlay the line plots
    overlay = curve_eu * curve_us * curve_eu_us
    overlay.opts(show_legend = True, legend_position='top_left', width=1200, height=600)
    return overlay


def create_bar_charts(region, **kwargs):
  if region=='eu':
    sliced = sorted_ent_type_freq_map_eu
    return hv.Bars(sliced, hv.Dimension('Entity Types'), 'Frequency').opts( framewise=True, xrotation=45,width=1200, height=600)
  elif region=='us':
    sliced = sorted_ent_type_freq_map_us
    return hv.Bars(sliced, hv.Dimension('Entity Types'), 'Frequency').opts(framewise=True, xrotation=45,width=1200, height=600)
  elif region=='eu_us':
    sliced = sorted_ent_type_freq_map_eu_us
    return hv.Bars(sliced, hv.Dimension('Entity Types'), 'Frequency').opts(framewise=True, xrotation=45,width=1200, height=600)



# Define a function to generate Curve based on selected values
def generate_entity_curves(region_value, type_value, **kwargs):
  if region_value=='eu':
    top20Ents = ent_freq_maps_eu[type_value]
    curveList = []
    for ent in top20Ents:
      entityTriples =  top_type_filtered_triples_eu[type_value][(top_type_filtered_triples_eu[type_value]['subjEntityLinks']==ent[0]) | (top_type_filtered_triples_eu[type_value]['objEntityLinks']==ent[0])]
      entityTriples_time_indexed = entityTriples.set_index(pd.DatetimeIndex(entityTriples['timestamp']), inplace=False)
      del entityTriples_time_indexed['timestamp']
      entityTriples_time_indexed_resampled = entityTriples_time_indexed.resample("Y").count()
      #print(entityTriples_time_indexed_resampled)
      entityTriples_time_indexed_resampled = entityTriples_time_indexed_resampled.reindex(dna_healthtech_articles_eu_time_indexed_resampled.index, fill_value=0)
      curve = hv.Curve((entityTriples_time_indexed_resampled.index, (entityTriples_time_indexed_resampled['doc_id']/dna_healthtech_articles_eu_time_indexed_resampled['ids'])), 'Time', 'Key Entity Occurrence', label=ent[0])
      curve.opts(autorange='y')
      #curve.opts(logy=True)
      curveList.append(curve)
    overlay = hv.Overlay(curveList)
    overlay.opts(legend_muted=False, legend_cols=4, show_legend = True, legend_position='top_left', fontsize={'legend':13},width=1200, height=800)
    return overlay

  elif region_value=='us':
    top20Ents = ent_freq_maps_us[type_value]
    curveList = []
    for ent in top20Ents:
      entityTriples =  top_type_filtered_triples_us[type_value][(top_type_filtered_triples_us[type_value]['subjEntityLinks']==ent[0]) | (top_type_filtered_triples_us[type_value]['objEntityLinks']==ent[0])]
      entityTriples_time_indexed = entityTriples.set_index(pd.DatetimeIndex(entityTriples['timestamp']), inplace=False)
      del entityTriples_time_indexed['timestamp']
      entityTriples_time_indexed_resampled = entityTriples_time_indexed_resampled.reindex(dna_healthtech_articles_us_time_indexed_resampled.index, fill_value=0)
      curve = hv.Curve((entityTriples_time_indexed_resampled.index, (entityTriples_time_indexed_resampled['doc_id']/dna_healthtech_articles_us_time_indexed_resampled['ids'])), 'Time', 'Key Entity Occurrence', label=ent[0])
      curve.opts(autorange='y')
      curveList.append(curve)
    overlay = hv.Overlay(curveList)
    overlay.opts(legend_muted=False, legend_cols=4, show_legend = True, legend_position='top_left', fontsize={'legend':13},width=1200, height=800)
    return overlay

  elif region_value=='eu_us':
    top20Ents = ent_freq_maps_eu_us[type_value]
    curveList = []
    for ent in top20Ents:
      entityTriples =  top_type_filtered_triples_eu_us[type_value][(top_type_filtered_triples_eu_us[type_value]['subjEntityLinks']==ent[0]) | (top_type_filtered_triples_eu_us[type_value]['objEntityLinks']==ent[0])]
      entityTriples_time_indexed = entityTriples.set_index(pd.DatetimeIndex(entityTriples['timestamp']), inplace=False)
      del entityTriples_time_indexed['timestamp']
      entityTriples_time_indexed_resampled = entityTriples_time_indexed_resampled.reindex(dna_healthtech_articles_eu_us_time_indexed_resampled.index, fill_value=0)
      curve = hv.Curve((entityTriples_time_indexed_resampled.index, (entityTriples_time_indexed_resampled['doc_id']/dna_healthtech_articles_eu_us_time_indexed_resampled['ids'])), 'Time', 'Key Entity Occurrence', label=ent[0])
      curve.opts(autorange='y')
      curveList.append(curve)
    overlay = hv.Overlay(curveList)
    overlay.opts(legend_muted=False, legend_cols=4, show_legend = True, legend_position='top_left', fontsize={'legend':13},width=1200, height=800)
    return overlay


############################# WIDGETS & CALLBACK ###########################################

def filter_data0(df, min_value):
    filtered_df = df[df['value'] >= min_value]
    return filtered_df


def plot_chord0_new(df,min_value):
    filtered_df = filter_data0(df, min_value)
  # Create a Holoviews Dataset for nodes
    nodes = hv.Dataset(filtered_df, 'index')
    nodes.data.head()
    chord = hv.Chord(filtered_df, ['source', 'target'], ['value'])
    return chord.opts(opts.Chord(cmap='Category20', edge_cmap='Category20', label_text_color="white",  node_color = hv.dim('index').str(),  edge_color = hv.dim('source').str(), labels = 'index', tools=['hover'],   width=800, height=800))


def retrieveRegionTypes(region):
  if region == 'eu':
    return top_type_filtered_eu
  elif region == 'us':
    return top_type_filtered_us
  elif region == 'eu_us':
    return top_type_filtered_eu_us


def filter_region(region):
    if region == 'eu':
        region_grouping = grouping_filtered[grouping_filtered['region'] == 'eu']
    elif region == 'us':
        region_grouping = grouping_filtered[grouping_filtered['region'] == 'us']
    elif region == 'eu_us':
        region_grouping = grouping_filtered[grouping_filtered['region'] == 'eu_us']

    #print(len(region_grouping))
    # Define range for minimum value slider
    min_value_range = region_grouping['value'].unique()
    min_value_range.sort()

    # Define HoloMap with minimum value and attribute as key dimensions
    holomap = hv.HoloMap({min_value: plot_chord0_new(region_grouping, min_value)
                          for min_value in min_value_range},
                         kdims=['Show triples with support greater than']
                         )
    return holomap


# Define a function to generate Entity List RadioButtonGroup based on Region selection
def generate_radio_buttons(value):
    if value == 'eu':
        return pn.widgets.RadioButtonGroup(options=retrieveRegionTypes(value), value='DBpedia:Company', name='eu', orientation='vertical')
    elif value == 'us':
        return pn.widgets.RadioButtonGroup(options=retrieveRegionTypes(value), value='DBpedia:Disease', name='us', orientation='vertical')
    elif value == 'eu_us':
        return pn.widgets.RadioButtonGroup(options=retrieveRegionTypes(value), value='DBpedia:Person', name='eu_us', orientation='vertical')



# https://tabler-icons.io/
button1 = pn.widgets.Button(name="Introduction", button_type="warning", icon="file-info", styles={"width": "100%"})
button2 = pn.widgets.Button(name="Health Tech News Ratio", button_type="warning",  icon="chart-histogram", styles={"width": "100%"})
button3 = pn.widgets.Button(name="Top Entity Types", button_type="warning", icon="chart-bar", styles={"width": "100%"})
button4 = pn.widgets.Button(name="Top Key Entities", button_type="warning", icon="chart-dots-filled", styles={"width": "100%"})
button5 = pn.widgets.Button(name="Entity Chord Diagrams", button_type="warning", icon="chart-dots-filled", styles={"width": "100%"})


region1 = pn.widgets.RadioButtonGroup(name='### Select News Region', options=regions)


# Initial RadioButtonGroup
radio_buttons_regions =  pn.widgets.RadioButtonGroup(options=regions,value='eu',name='Select region')
# Generate initial dynamic RadioButtonGroup
radio_buttons_types  = generate_radio_buttons(radio_buttons_regions.value)



# Define a callback function to update the panel dynamically
def update_radio_group(event):
  #print(event.new)
  #print(retrieveRegionTypes(event.new))
  radio_buttons_types.options = retrieveRegionTypes(event.new)


# bind the function to the widget(s)
dmap2 = hv.DynamicMap(pn.bind(generate_entity_curves, radio_buttons_regions,radio_buttons_types))
# Bind the selected value of the first RadioButtonGroup to update the second RadioButtonGroup
radio_buttons_regions.param.watch(update_radio_group, 'value')

# Define the callback function to update the HoloMap
def update_holomap(event):
    initial_holomap.object = filter_region(event.new)

region_radio_button = pn.widgets.RadioButtonGroup(options=regions, value='eu', name='Select Region')

# Create the initial HoloMap
initial_holomap = filter_region(region_radio_button.value)

# Bind the callback function to the value change event of the RadioButton widget
region_radio_button.param.watch(update_holomap, 'value')



def show_page(page_key):
    main_area.clear()
    main_area.append(mapping[page_key])

button1.on_click(lambda event: show_page("Page1"))
button2.on_click(lambda event: show_page("Page2"))
button3.on_click(lambda event: show_page("Page3"))
button4.on_click(lambda event: show_page("Page4"))
button5.on_click(lambda event: show_page("Page5"))


### CREATE PAGE LAYOUTS

def CreatePage1():
    return pn.Column(pn.pane.Markdown("""

This is a dashboard for a News Analysis project regarding Digital Health technology. The source data consists of around 7.8 million English-language news articles gathered from the **Dow Jones Data, News, and Analytics (DNA)**
platform (https://www.dowjones.com/professional/developer-platform/) covering a timeframe from September 1987 through December 2023. The news items text content is copyrighted and cannot be shared within this project.

Some of the data analytics visualizations show here come from a Knowledge Graph automatically extracted from DNA news sources. A Virtuoso SPARQL endpoint to this graph (named 'DHNEWS KG') is set up at the 
URL: https://api-vast.jrc.service.ec.europa.eu/sparql/


---------------------------

## 1. Health Tech News Ratio 
In the Health Tech News Ratio panel we present the month-sampled time series depicting the proportion of 97k news articles con-
cerning Digital Health, as identified by a text classifier, out of the total number of English language DNA news articles pertaining to Europe and the US


### 2. Top Entity Types 
The Top Entity Types bar plots in the dashboard show the predominant DBpedia-inherited entity types within the graph for triples tagged with Europe, US, and EU-US region codes via their article support.

## 3. Top Key Entities 
The Top Key Entities plots track the occurrence of several key entities per year, where occurrence means the entity is either the Subject or Object of an extracted triple in the KG.

## 4. Entity Chord Diagrams  
Entity Chord Diagrams represent the most frequently connected entity pairs within the KG through chord illustrations, serving as both Subjects and Objects of predicative triples. 
The size of the chords corresponds to the support of the depicted relations.
""", width=800), align="center")

def CreatePage2():
    return pn.Column(
        pn.pane.Markdown("## Health Tech News Ratio "),
        create_curve_chart(),
        align="center",
    )

def CreatePage3():
    return pn.Column(
    region1,
        pn.bind(create_bar_charts, region1),
        align="center",
    )

def CreatePage4():
    return pn.Column(
        pn.pane.Markdown("## Top Key Entities "),
        pn.Row(pn.Column(radio_buttons_regions, radio_buttons_types), dmap2),
        align="center", )

def CreatePage5():
    return pn.Column(
        pn.pane.Markdown("## Entity Chord Diagrams "),
        pn.Row(region_radio_button, pn.bind(filter_region, region_radio_button)),
        align="center", )

mapping = {
    "Page1": CreatePage1(),
    "Page2": CreatePage2(),
    "Page3": CreatePage3(),
    "Page4": CreatePage4(),
    "Page5": CreatePage5(),
}

#################### SIDEBAR LAYOUT ##########################
sidebar = pn.Column(pn.pane.Markdown("## Pages"), button1,button2,button3,
                    button4,
                    button5,
					styles={"width": "100%", "padding": "15px"})

#################### MAIN AREA LAYOUT ##########################
main_area = pn.Column(mapping["Page1"], styles={"width":"100%"})

###################### APP LAYOUT ##############################
template = pn.template.BootstrapTemplate(
    title=" Digital Health in the News: Analytics Dashboard ",
    sidebar=[sidebar],
    main=[main_area],
    header_background="black",
    #site="Charting the Landscape of Digital Health",
    theme=pn.template.DarkTheme,
    sidebar_width=250, ## Default is 330
    busy_indicator=pn.indicators.BooleanStatus(value=True),
)

### DEPLOY APP

# Serve the Panel app
template.servable()