app.py

# -*- coding: utf-8 -*-
"""
NPS Assurance — Gradio (Paste-only)
- Entrée: verbatims collés (1 ligne = 1 verbatim, score NPS optionnel après un séparateur, ex: "|")
- Sorties: émotion (pos/neutre/neg), thématiques, occurrences, résumé Markdown, graphiques Plotly
- IA (facultatif): OpenAI pour sentiment/thèmes/synthèse. Sans clé, fallback HuggingFace (si installé) puis règles lexicales.
- Déployable tel quel sur Hugging Face Spaces (app_file = app.py)
"""

import os, re, json, collections, tempfile, zipfile
from typing import List, Dict, Optional
import pandas as pd
from unidecode import unidecode
import gradio as gr
import plotly.express as px
import plotly.graph_objects as go

# ---------------- Thésaurus ASSURANCE ----------------
THEMES = {
    "Remboursements santé":[r"\bremboursement[s]?\b", r"\bt[eé]l[eé]transmission\b", r"\bno[eé]mie\b",
        r"\bprise\s*en\s*charge[s]?\b", r"\btaux\s+de\s+remboursement[s]?\b", r"\b(ameli|cpam)\b",
        r"\bcompl[eé]mentaire\s+sant[eé]\b", r"\bmutuelle\b", r"\battestation[s]?\b", r"\bcarte\s+(mutuelle|tiers\s*payant)\b"],
    "Tiers payant / Réseau de soins":[r"\btiers\s*payant\b", r"\br[ée]seau[x]?\s+de\s+soins\b",
        r"\b(optique|dentaire|hospitalisation|pharmacie)\b", r"\bitelis\b", r"\bsant[eé]clair\b", r"\bkalixia\b"],
    "Sinistres / Indemnisation":[r"\bsinistre[s]?\b", r"\bindemni(sation|ser)\b", r"\bexpertis[ea]\b",
        r"\bd[eé]claration\s+de\s+sinistre\b", r"\bconstat\b", r"\bbris\s+de\s+glace\b", r"\bassistance\b", r"\bd[ée]pannage\b"],
    "Adhésion / Contrat":[r"\badh[eé]sion[s]?\b", r"\bsouscription[s]?\b", r"\baffiliation[s]?\b", r"\bcontrat[s]?\b",
        r"\bavenant[s]?\b", r"\bcarence[s]?\b", r"\brenouvellement[s]?\b", r"\br[eé]siliation[s]?\b"],
    "Garanties / Exclusions / Franchise":[r"\bgarantie[s]?\b", r"\bexclusion[s]?\b", r"\bplafond[s]?\b",
        r"\bfranchise[s]?\b", r"\bconditions\s+g[eé]n[eé]rales\b", r"\bnotice\b"],
    "Cotisations / Facturation":[r"\bcotisation[s]?\b", r"\bpr[eé]l[eè]vement[s]?\b", r"\bech[eé]ancier[s]?\b",
        r"\bfacture[s]?\b", r"\berreur[s]?\s+de\s+facturation\b", r"\bremboursement[s]?\b", r"\bRIB\b", r"\bIBAN\b"],
    "Délais & Suivi dossier":[r"\bd[eé]lai[s]?\b", r"\btraitement[s]?\b", r"\bsuivi[s]?\b", r"\brelance[s]?\b", r"\bretard[s]?\b"],
    "Espace client / App / Connexion":[r"\bespace\s+client\b", r"\bapplication\b", r"\bapp\b", r"\bsite\b",
        r"\bconnexion\b", r"\bidentifiant[s]?\b", r"\bmot\s+de\s+passe\b", r"\bpaiement\s+en\s+ligne\b",
        r"\bbogue[s]?\b", r"\bbug[s]?\b", r"\bnavigation\b", r"\binterface\b", r"\bUX\b"],
    "Support / Conseiller":[r"\bSAV\b", r"\bservice[s]?\s+client[s]?\b", r"\bconseiller[s]?\b",
        r"\b[rR][eé]ponse[s]?\b", r"\bjoignable[s]?\b", r"\brapp?el\b"],
    "Communication / Transparence":[r"\binformation[s]?\b", r"\bcommunication\b", r"\btransparence\b",
        r"\bclart[eé]\b", r"\bcourrier[s]?\b", r"\bmail[s]?\b", r"\bnotification[s]?\b"],
    "Prix":[r"\bprix\b", r"\bcher[s]?\b", r"\bco[uû]t[s]?\b", r"\btarif[s]?\b",
        r"\bcomp[eé]titif[s]?\b", r"\babusif[s]?\b", r"\bbon\s+rapport\s+qualit[eé]\s*prix\b"],
    "Offre / Gamme":[r"\boffre[s]?\b", r"\bgamme[s]?\b", r"\bdisponibilit[eé][s]?\b", r"\bdevis\b", r"\bchoix\b", r"\bcatalogue[s]?\b"],
    "Produit/Qualité":[r"\bqualit[eé]s?\b", r"\bfiable[s]?\b", r"\bconforme[s]?\b", r"\bnon\s+conforme[s]?\b",
        r"\bd[eé]fectueux?[es]?\b", r"\bperformant[e]?[s]?\b"],
    "Agence / Accueil":[r"\bagence[s]?\b", r"\bboutique[s]?\b", r"\baccueil\b", r"\bconseil[s]?\b", r"\battente\b", r"\bcaisse[s]?\b"],
}

# --------------- Sentiment (fallback règles) ---------------
POS_WORDS = {"bien":1.0,"super":1.2,"parfait":1.4,"excellent":1.5,"ravi":1.2,"satisfait":1.0,
             "rapide":0.8,"efficace":1.0,"fiable":1.0,"simple":0.8,"facile":0.8,"clair":0.8,"conforme":0.8,
             "sympa":0.8,"professionnel":1.0,"réactif":1.0,"reactif":1.0,"compétent":1.0,"competent":1.0,
             "top":1.2,"recommande":1.2,"recommandé":1.2,"bon":0.8}
NEG_WORDS = {"mauvais":-1.2,"horrible":-1.5,"nul":-1.2,"lent":-0.8,"cher":-0.9,"arnaque":-1.5,
             "déçu":-1.2,"decu":-1.2,"incompétent":-1.3,"bug":-0.9,"bogue":-0.9,"problème":-1.0,
             "probleme":-1.0,"attente":-0.6,"retard":-0.9,"erreur":-1.0,"compliqué":-0.8,"complique":-0.8,
             "défectueux":-1.3,"defectueux":-1.3,"non conforme":-1.2,"impossible":-1.0,"difficile":-0.7}
NEGATIONS = [r"\bpas\b", r"\bjamais\b", r"\bplus\b", r"\baucun[e]?\b", r"\brien\b", r"\bni\b", r"\bgu[eè]re\b"]
INTENSIFIERS = [r"\btr[eè]s\b", r"\bvraiment\b", r"\bextr[eê]mement\b", r"\bhyper\b"]
DIMINISHERS  = [r"\bun[e]?\s+peu\b", r"\bassez\b", r"\bplut[oô]t\b", r"\bl[eé]g[eè]rement\b"]
INTENSIFIER_W, DIMINISHER_W = 1.5, 0.7

# --------------- OpenAI (optionnel) ---------------
OPENAI_AVAILABLE = False
try:
    from openai import OpenAI
    _client = OpenAI()  # clé via OPENAI_API_KEY (en secret HF)
    OPENAI_AVAILABLE = True
except Exception:
    OPENAI_AVAILABLE = False

# ---------------- Utils ----------------
def normalize(t:str)->str:
    if not isinstance(t,str): return ""
    return re.sub(r"\s+"," ",t.strip())

def to_analyzable(t:str)->str:
    return unidecode(normalize(t.lower()))

def window_has(patterns:List[str], toks:List[str], i:int, w:int=3)->bool:
    s=max(0,i-w); e=min(len(toks),i+w+1); win=" ".join(toks[s:e])
    return any(re.search(p,win) for p in patterns)

def lexical_sentiment_score(text:str)->float:
    toks = to_analyzable(text).split(); score=0.0
    for i,t in enumerate(toks):
        base = POS_WORDS.get(t,0.0) or NEG_WORDS.get(t,0.0)
        if not base and i<len(toks)-1:
            bi=f"{t} {toks[i+1]}"; base = NEG_WORDS.get(bi,0.0)
        if base:
            w=1.0
            if window_has(INTENSIFIERS,toks,i): w*=INTENSIFIER_W
            if window_has(DIMINISHERS,toks,i):  w*=DIMINISHER_W
            if window_has(NEGATIONS,toks,i):    base*=-1
            score+=base*w
    return max(min(score,4.0),-4.0)

def lexical_sentiment_label(s:float)->str:
    return "positive" if s>=0.3 else ("negatif" if s<=-0.3 else "neutre")

def detect_themes_regex(text:str):
    t=to_analyzable(text); counts={}
    for th,pats in THEMES.items():
        c=sum(len(re.findall(p,t)) for p in pats)
        if c>0: counts[th]=c
    return list(counts.keys()), counts

def nps_bucket(s):
    try:
        v=int(s)
    except:
        return "inconnu"
    return "promoter" if v>=9 else ("passive" if v>=7 else ("detractor" if v>=0 else "inconnu"))

def compute_nps(series):
    vals=[]
    for x in series.dropna().tolist():
        try:
            v=int(x)
            if 0<=v<=10: vals.append(v)
        except: pass
    if not vals: return None
    tot=len(vals); pro=sum(1 for v in vals if v>=9); det=sum(1 for v in vals if v<=6)
    return 100.0*(pro/tot - det/tot)

def anonymize(t:str)->str:
    if not isinstance(t,str): return ""
    t=re.sub(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}","[email]",t)
    t=re.sub(r"\b(?:\+?\d[\s.-]?){7,}\b","[tel]",t)
    return t

# --------- Coller du texte → DataFrame ----------
def df_from_pasted(text:str, sep="|", has_score=False) -> pd.DataFrame:
    lines = [l.strip() for l in (text or "").splitlines() if l.strip()]
    rows = []
    for i, line in enumerate(lines, 1):
        if has_score and sep in line:
            verb, score = line.split(sep, 1)
            rows.append({"id": i, "comment": verb.strip(), "nps_score": pd.to_numeric(score.strip(), errors="coerce")})
        else:
            rows.append({"id": i, "comment": line.strip(), "nps_score": None})
    return pd.DataFrame(rows)

# --------- OpenAI helpers (optionnels) ----------
def openai_json(model:str, system:str, user:str, temperature:float=0.0) -> Optional[dict]:
    if not OPENAI_AVAILABLE: return None
    try:
        resp = _client.chat.completions.create(
            model=model, temperature=temperature,
            messages=[{"role":"system","content":system},{"role":"user","content":user}],
        )
        txt = resp.choices[0].message.content.strip()
        m = re.search(r"\{.*\}", txt, re.S)
        return json.loads(m.group(0) if m else txt)
    except Exception:
        return None

def oa_sentiment(comment:str, model:str, temperature:float=0.0) -> Optional[dict]:
    system = "Tu es un classifieur FR. Réponds strictement en JSON."
    user = f'Texte: {comment}\nDonne "label" parmi ["positive","neutre","negatif"] et "score" entre -4 et 4. JSON.'
    return openai_json(model, system, user, temperature)

def oa_themes(comment:str, model:str, temperature:float=0.0) -> Optional[dict]:
    system = "Tu maps le texte client vers un thésaurus assurance. Réponds strictement en JSON."
    user = f"Texte: {comment}\nThésaurus: {json.dumps(list(THEMES.keys()), ensure_ascii=False)}\nRetourne {{'themes': [...], 'counts': {{...}}}}"
    return openai_json(model, system, user, temperature)

def oa_summary(nps:Optional[float], dist:Dict[str,int], themes_df:pd.DataFrame, model:str, temperature:float=0.2) -> Optional[str]:
    system = "Tu es un analyste CX FR. Donne une synthèse courte et actionnable en Markdown."
    top = [] if themes_df is None else themes_df.head(6).to_dict(orient="records")
    user = f"Données: NPS={None if nps is None else round(nps,1)}, Répartition={dist}, Thèmes={json.dumps(top, ensure_ascii=False)}"
    j = openai_json(model, system, user, temperature)
    if isinstance(j, dict) and "text" in j: return j["text"]
    if isinstance(j, dict): return ' '.join(str(v) for v in j.values())
    return None

# --------- Graphiques ----------
def fig_nps_gauge(nps: Optional[float]) -> go.Figure:
    v = 0.0 if nps is None else float(nps)
    return go.Figure(go.Indicator(mode="gauge+number", value=v,
                                  gauge={"axis":{"range":[-100,100]}, "bar":{"thickness":0.3}},
                                  title={"text":"NPS (−100 à +100)"}))

def fig_sentiment_bar(dist: Dict[str,int]) -> go.Figure:
    order = ["negatif","neutre","positive"]
    x = [o for o in order if o in dist]; y = [dist.get(o,0) for o in x]
    return px.bar(x=x, y=y, labels={"x":"Sentiment","y":"Nombre"}, title="Répartition des émotions")

def fig_top_themes(themes_df: pd.DataFrame, k: int) -> go.Figure:
    if themes_df is None or themes_df.empty: return go.Figure()
    d = themes_df.head(k); fig = px.bar(d, x="theme", y="total_mentions", title=f"Top {k} thèmes — occurrences")
    fig.update_layout(xaxis_tickangle=-30); return fig

def fig_theme_balance(themes_df: pd.DataFrame, k: int) -> go.Figure:
    if themes_df is None or themes_df.empty: return go.Figure()
    d = themes_df.head(k)
    d2 = d.melt(id_vars=["theme"], value_vars=["verbatims_pos","verbatims_neg"], var_name="type", value_name="count")
    d2["type"] = d2["type"].map({"verbatims_pos":"Positifs","verbatims_neg":"Négatifs"})
    fig = px.bar(d2, x="theme", y="count", color="type", barmode="stack", title=f"Top {k} thèmes — balance Pos/Neg")
    fig.update_layout(xaxis_tickangle=-30); return fig

# --------- Analyse principale ----------
def analyze_text(pasted_txt, has_sc, sep_chr,
                 do_anonymize, use_oa_sent, use_oa_themes, use_oa_summary,
                 oa_model, oa_temp, top_k):

    df = df_from_pasted(pasted_txt or "", sep=sep_chr or "|", has_score=bool(has_sc))
    if df.empty:
        raise gr.Error("Colle au moins un verbatim (une ligne).")

    if do_anonymize:
        df["comment"]=df["comment"].apply(anonymize)

    if (use_oa_sent or use_oa_themes or use_oa_summary) and not OPENAI_AVAILABLE:
        raise gr.Error("OpenAI non dispo : installe `openai` et définis OPENAI_API_KEY, ou décoche les options OpenAI.")

    # HF sentiment (optionnel)
    HF_AVAILABLE=False
    try:
        from transformers import pipeline
        hf_pipe = pipeline("text-classification",
                           model="cmarkea/distilcamembert-base-sentiment",
                           tokenizer="cmarkea/distilcamembert-base-sentiment")
        HF_AVAILABLE=True
    except Exception:
        HF_AVAILABLE=False
    def hf_sent(text:str):
        if not HF_AVAILABLE or not text.strip(): return None
        try:
            res=hf_pipe(text); lab=str(res[0]["label"]).lower(); p=float(res[0].get("score",0.5))
            if "1" in lab or "2" in lab: return {"label":"negatif","score":-4*p}
            if "3" in lab:               return {"label":"neutre","score":0.0}
            return {"label":"positive","score":4*p}
        except Exception:
            return None

    rows=[]
    theme_agg=collections.defaultdict(lambda:{"mentions":0,"pos":0,"neg":0})
    used_hf=False; used_oa=False

    for _, r in df.iterrows():
        cid=r["id"]; comment=normalize(str(r["comment"]))

        # Sentiment: OpenAI -> HF -> règles
        sent=None
        if use_oa_sent:
            sent=oa_sentiment(comment, oa_model, oa_temp); used_oa = used_oa or bool(sent)
        if not sent:
            hf=hf_sent(comment)
            if hf: sent=hf; used_hf=True
        if not sent:
            s=float(lexical_sentiment_score(comment))
            sent={"label":lexical_sentiment_label(s),"score":s}

        # Thèmes: regex (+ fusion OpenAI)
        themes, counts = detect_themes_regex(comment)
        if use_oa_themes:
            tjson=oa_themes(comment, oa_model, oa_temp)
            if isinstance(tjson, dict):
                used_oa=True
                for th, c in (tjson.get("counts",{}) or {}).items():
                    if th in THEMES and int(c) > 0:
                        counts[th] = max(counts.get(th, 0), int(c))
                themes = [th for th, c in counts.items() if c > 0]

        bucket = nps_bucket(r.get("nps_score", None))

        for th, c in counts.items():
            theme_agg[th]["mentions"] += c
            if sent["label"] == "positive":
                theme_agg[th]["pos"] += 1
            elif sent["label"] == "negatif":
                theme_agg[th]["neg"] += 1

        rows.append({
            "id": cid, "nps_score": r.get("nps_score", None), "nps_bucket": bucket,
            "comment": comment,
            "sentiment_score": round(float(sent["score"]), 3),
            "sentiment_label": sent["label"],
            "sentiment_source": "openai" if (use_oa_sent and used_oa) else ("huggingface" if used_hf else "rules"),
            "themes": ", ".join(themes) if themes else "",
            "theme_counts_json": json.dumps(counts, ensure_ascii=False)
        })

    out_df=pd.DataFrame(rows)
    nps=compute_nps(df["nps_score"])  # peut être None si pas de scores
    dist=out_df["sentiment_label"].value_counts().to_dict()

    # Stats par thème
    trs=[]
    for th, d in theme_agg.items():
        trs.append({"theme":th,"total_mentions":int(d["mentions"]),
                    "verbatims_pos":int(d["pos"]),"verbatims_neg":int(d["neg"]),
                    "net_sentiment":int(d["pos"]-d["neg"])})
    themes_df=pd.DataFrame(trs).sort_values(["total_mentions","net_sentiment"],ascending=[False,False])

    # Synthèse texte
    method = "OpenAI + HF + règles" if (use_oa_sent and used_hf) else ("OpenAI + règles" if use_oa_sent else ("HF + règles" if used_hf else "Règles"))
    lines=[ "# Synthèse NPS & ressentis clients",
            f"- **Méthode** : {method}",
            f"- **NPS global** : {nps:.1f}" if nps is not None else "- **NPS global** : n/a" ]
    if dist:
        tot=sum(dist.values()); pos=dist.get("positive",0); neg=dist.get("negatif",0); neu=dist.get("neutre",0)
        lines.append(f"- **Répartition émotions** : positive {pos}/{tot}, neutre {neu}/{tot}, négative {neg}/{tot}")
    if not themes_df.empty:
        lines.append("\n## Thèmes les plus cités")
        for th,m in themes_df.head(5)[["theme","total_mentions"]].values.tolist():
            lines.append(f"- **{th}** : {m} occurrence(s)")
    summary_md="\n".join(lines)

    if use_oa_summary:
        md = oa_summary(nps, dist, themes_df, oa_model, oa_temp)
        if md: summary_md = md + "\n\n---\n" + summary_md

    # Fichiers export
    tmpdir=tempfile.mkdtemp(prefix="nps_gradio_")
    enriched=os.path.join(tmpdir,"enriched_comments.csv")
    themes=os.path.join(tmpdir,"themes_stats.csv")
    summ=os.path.join(tmpdir,"summary.md")
    out_df.to_csv(enriched,index=False,encoding="utf-8-sig")
    themes_df.to_csv(themes,index=False,encoding="utf-8-sig")
    with open(summ,"w",encoding="utf-8") as f: f.write(summary_md)
    zip_path=os.path.join(tmpdir,"nps_outputs.zip")
    with zipfile.ZipFile(zip_path,"w",zipfile.ZIP_DEFLATED) as z:
        z.write(enriched,arcname="enriched_comments.csv")
        z.write(themes,arcname="themes_stats.csv")
        z.write(summ,arcname="summary.md")

    # Graphiques
    fig_gauge = fig_nps_gauge(nps)
    fig_emots = fig_sentiment_bar(dist)
    k = max(1, int(top_k or 10))
    fig_top   = fig_top_themes(themes_df, k)
    fig_bal   = fig_theme_balance(themes_df, k)

    # Panneaux (rapide)
    def make_panels(dfT: pd.DataFrame):
        if dfT is None or dfT.empty: return "—","—","—"
        pos_top = dfT.sort_values(["verbatims_pos","total_mentions"], ascending=[False,False]).head(4)
        neg_top = dfT.sort_values(["verbatims_neg","total_mentions"], ascending=[False,False]).head(4)
        def bullets(df, col, label):
            lines=[f"**{label}**"]
            for _, r in df.iterrows(): lines.append(f"- **{r['theme']}** — {int(r[col])} verbatims")
            return "\n".join(lines)
        ench_md = bullets(pos_top, "verbatims_pos", "Points d’enchantement")
        irr_md  = bullets(neg_top, "verbatims_neg", "Irritants")
        RECO_RULES = {
            "Délais & Suivi dossier": "Réduire les délais (SLA), suivi proactif.",
            "Cotisations / Facturation": "Clarifier factures, alerter anomalies.",
            "Espace client / App / Connexion": "Corriger login/MDP, QA navigateurs.",
            "Support / Conseiller": "Améliorer joignabilité, scripts, rappel auto.",
            "Communication / Transparence": "Notifications étapes clés, messages clairs.",
            "Sinistres / Indemnisation": "Transparence délais + suivi dossier.",
        }
        rec_lines=["**Recommandations**"]
        for _, r in neg_top.iterrows():
            rec_lines.append(f"- **{r['theme']}** — {RECO_RULES.get(r['theme'],'Plan d’action dédié')}")
        return ench_md, irr_md, "\n".join(rec_lines)

    ench_md, irr_md, reco_md = make_panels(themes_df)

    return (summary_md, themes_df.head(100), out_df.head(200), [enriched, themes, summ, zip_path],
            ench_md, irr_md, reco_md, fig_gauge, fig_emots, fig_top, fig_bal)

# ---------------- UI ----------------
with gr.Blocks(title="NPS — Analyse (Assurance)") as demo:
    gr.Markdown("## 🔎 NPS — Analyse sémantique (Assurance)\nColle tes verbatims (1 par ligne). Option: score NPS après un `|`.")

    with gr.Column():
        pasted = gr.Textbox(label="Verbatims (un par ligne)", lines=10,
                            placeholder="Exemple :\nRemboursement rapide, télétransmission OK | 10\nConnexion impossible à l’app | 3\nDélais corrects | 7")
        with gr.Row():
            has_score = gr.Checkbox(label="J’ai un score NPS par ligne", value=False)
            sep = gr.Textbox(label="Séparateur score", value="|", scale=1)

    with gr.Row():
        anon=gr.Checkbox(label="Anonymiser emails / téléphones", value=True)
        use_oa_sent=gr.Checkbox(label="OpenAI pour le sentiment", value=False)
        use_oa_themes=gr.Checkbox(label="OpenAI pour les thèmes", value=False)
        use_oa_summary=gr.Checkbox(label="OpenAI pour la synthèse", value=False)

    with gr.Row():
        oa_model=gr.Textbox(label="Modèle OpenAI", value="gpt-4o-mini")
        oa_temp=gr.Slider(label="Température", minimum=0.0, maximum=1.0, value=0.1, step=0.1)
        top_k=gr.Slider(label="Top thèmes (K) pour les graphes", minimum=5, maximum=20, value=10, step=1)
        run=gr.Button("Lancer l'analyse", variant="primary")

    with gr.Row():
        ench_panel=gr.Markdown(); irr_panel=gr.Markdown(); reco_panel=gr.Markdown()

    summary=gr.Markdown(label="Synthèse")
    themes_table=gr.Dataframe(label="Thèmes — statistiques")
    enriched_table=gr.Dataframe(label="Verbatims enrichis (aperçu)")
    files_out=gr.Files(label="Téléchargements (CSV & ZIP)")

    with gr.Row():
        plot_nps = gr.Plot(label="NPS — Jauge")
        plot_sent= gr.Plot(label="Répartition des émotions")
    with gr.Row():
        plot_top  = gr.Plot(label="Top thèmes — occurrences")
        plot_bal  = gr.Plot(label="Top thèmes — balance Pos/Neg")

    run.click(
        analyze_text,
        inputs=[pasted, has_score, sep, anon, use_oa_sent, use_oa_themes, use_oa_summary, oa_model, oa_temp, top_k],
        outputs=[summary, themes_table, enriched_table, files_out, ench_panel, irr_panel, reco_panel, plot_nps, plot_sent, plot_top, plot_bal]
    )

if __name__=="__main__":
    demo.launch(share=False, show_api=False)