app.py

import gradio as gr
import spacy
from spacy import displacy
from cefrpy import CEFRSpaCyAnalyzer, CEFRLevel

MODEL = "en_core_web_sm"

ALL_ENTS = [
    'CARDINAL', 'DATE', 'EVENT', 'FAC', 'GPE', 'LANGUAGE',
    'LAW', 'LOC', 'MONEY', 'NORP', 'ORDINAL', 'ORG', 'PERCENT',
    'PERSON', 'PRODUCT', 'QUANTITY', 'TIME', 'WORK_OF_ART'
]

DEFAULT_ENTITY_ITEMS_TO_SKIP = [
    'QUANTITY', 'MONEY', 'LANGUAGE', 'LAW',
    'WORK_OF_ART', 'PRODUCT', 'GPE',
    'ORG', 'FAC', 'PERSON'
]

TOKEN_ATTRIBUTES = [
    "Token",
    "POS",
    "Skipped",
    "Level",
    "Start",
    "End"
]

WORDLIST_HEADER = ["Word", "Pos", "CEFR", "Level"]

DEFAULT_WORDLIST_SLIDER_LEVEL = 4.0

DEFAULT_TEXT = """The world's oldest known recipe is for beer. It dates back to around 5,000 BC and was found in ancient Sumeria (modern-day Iraq).

Due to thermal expansion, the iron structure of the Eiffel Tower can expand in hot weather, making the tower grow by up to 6 inches (15 centimeters) in height.

Did you know that the word "antidisestablishmentarianism" is often cited as one of the longest non-technical words in the English language? It originated in the 19th century in Britain during debates over the disestablishment of the Church of England, and it refers to the opposition to the withdrawal of state support for an established church. This word has gained notoriety for its length and has been used as a challenge for spelling bees and word enthusiasts alike.

In 2006, a Coca-Cola employee offered to sell Coca-Cola secrets to Pepsi. Pepsi responded by notifying Coca-Cola, and the FBI set up a sting operation to catch the culprit.

Like humans, cows form strong social bonds and often have "best friends" within their herds. They display complex social behaviors, including grooming, playing, and even grieving when separated from their friends."""

DISPLACY_RENDER_OPTIONS = {
    "colors": {
        "A1": "#b0c4de",
        "A2": "#87ceeb",
        "B1": "#90ee90",
        "B2": "#adff2f",
        "C1": "#ffd700",
        "C2": "#ff9380",
        "SKIP": "#ffafed",
        "UNKNOWN": "#BCAAA4"
    }
}

ABBREVIATION_MAPPING = {
    "'m": "am",
    "'s": "is",
    "'re": "are",
    "'ve": "have",
    "'d": "had",
    "n't": "not",
    "'ll": "will"
}

LINKS_HTML = """
<p>
    &ensp;Github: <a href="https://github.com/Maximax67/cefrpy">link</a><br>
    &ensp;Docs: <a href="https://maximax67.github.io/cefrpy">link</a><br>
</p>
"""

CSS = """
h1 {
    padding-top: 5px;
    text-align: center;
    display:block;
}
"""

nlp = spacy.load(MODEL)

def get_dict_ents(text: str, tokens: list[tuple[str, str, bool, float, int, int]]) -> dict:
    ents = []

    for token in tokens:
        if token[3]:
            ents.append({
                "start": token[4],
                "end": token[5],
                "label": str(CEFRLevel(round(token[3])))
            })
        elif token[0].isalpha():
            ents.append({
                "start": token[4],
                "end": token[5],
                "label": "SKIP" if token[2] else "UNKNOWN"
            })

    dict_ents = {
        "text": text,
        "ents": ents
    }

    return dict_ents


def get_cefr_tokens(text: str, ents_to_skip: list[str]) -> list[tuple[str, str, bool, float, int, int]]:
    doc = nlp(text)
    text_analyzer = CEFRSpaCyAnalyzer(entity_types_to_skip=ents_to_skip, abbreviation_mapping=ABBREVIATION_MAPPING)
    tokens = text_analyzer.analize_doc(doc)

    return tokens


def get_html_visualization(text: str, tokens: list[tuple[str, str, bool, float, int, int]]) -> str:
    dict_ents = get_dict_ents(text, tokens)
    html = displacy.render(dict_ents, manual=True, style="ent", options=DISPLACY_RENDER_OPTIONS)

    return html


def get_wordlist_set(tokens: list[tuple[str, str, bool, float, int, int]],
                min_level: float) -> set[tuple[str, str, bool, float, int, int]]:
    filtered_tokens = set()
    for word, pos, _, level, _, _ in tokens:
        if level and level >= min_level:
            filtered_tokens.add((word.lower(), pos, str(CEFRLevel(round(level))), level))

    return filtered_tokens


def get_wordlist(tokens: list[tuple[str, str, bool, float, int, int]], min_level: float):
    wordlist_set = get_wordlist_set(tokens, min_level)
    wordlist = list(wordlist_set)
    wordlist.sort()

    return wordlist


def get_wordlist_from_dataframe(dataframe, min_level: float):
    return get_wordlist(dataframe.values, min_level)


def process_text(text: str, ents_to_skip: list[str] | None = DEFAULT_ENTITY_ITEMS_TO_SKIP, min_level: float = DEFAULT_WORDLIST_SLIDER_LEVEL) -> tuple[list[list], str]:
    tokens = get_cefr_tokens(text, ents_to_skip)
    html = get_html_visualization(text, tokens)
    wordlist = get_wordlist(tokens, min_level)

    return tokens, wordlist, html


initial_tokens, initial_wordlist, initial_html = process_text(DEFAULT_TEXT)

demo = gr.Blocks(css=CSS)

with demo:
    with gr.Row():
        with gr.Column():
            with gr.Column():
                with gr.Row():
                    gr.Markdown("# Gradio Demo: cefrpy")
                    gr.HTML(LINKS_HTML)

                with gr.Row():
                    text_input = gr.TextArea(
                        value=DEFAULT_TEXT,
                        interactive=True,
                        max_lines=500,
                        label="Input Text",
                        show_copy_button=True
                    )

                with gr.Row():
                    ent_input = gr.CheckboxGroup(
                        ALL_ENTS,
                        value=DEFAULT_ENTITY_ITEMS_TO_SKIP,
                        label="Entity types to skip CEFR"
                    )

                with gr.Row():
                    clear_button = gr.ClearButton(text_input)

                    render_button = gr.Button(
                        "Render",
                        variant="primary"
                    )

        with gr.Column():
            with gr.Row():
                gr.Markdown("# Words CEFR level visualization")

            with gr.Row():
                rendered_html = gr.HTML(initial_html)

    with gr.Row():
        with gr.Column():
            with gr.Row():
                tokens_output = gr.Dataframe(headers=TOKEN_ATTRIBUTES, value=initial_tokens, interactive=False)

        with gr.Column():
            with gr.Row():
                min_level_slider = gr.Slider(
                    minimum=1.0,
                    maximum=6.0, 
                    value=DEFAULT_WORDLIST_SLIDER_LEVEL,
                    step=0.02,
                    interactive=True,
                    label="Min level to generate word list"
                )

            with gr.Row():
                wordlist = gr.Dataframe(headers=WORDLIST_HEADER, value=initial_wordlist, interactive=False)

    render_button.click(
        process_text,
        inputs=[text_input, ent_input],
        outputs=[tokens_output, wordlist, rendered_html],
        api_name="process_text"
    )

    min_level_slider.release(
        get_wordlist_from_dataframe,
        inputs=[tokens_output, min_level_slider],
        outputs=[wordlist],
        api_name=False
    )

demo.launch(show_api=True)