simplify approach

app.py

@@ -1,21 +1,20 @@
-# import examples object from examples.json file
+import os
 import json
-
-with open("examples.json", "r") as f:
-    examples = json.load(f)
-
-from typing import Dict, Union
-from gliner import GLiNER
 import gradio as gr
+from gliner import GLiNER
 
-model = GLiNER.from_pretrained("urchade/gliner_medium-v2.1")
+# Load the common examples from the JSON file
+with open("examples.json", "r", encoding="utf-8") as f:
+    common_examples = json.load(f)
 
+# Utility function to merge adjacent entities (used in NuNER Zero)
 def merge_entities(entities):
     if not entities:
         return []
     merged = []
     current = entities[0]
     for next_entity in entities[1:]:
+        # Merge if same label and adjacent
         if next_entity['entity'] == current['entity'] and (next_entity['start'] == current['end'] + 1 or next_entity['start'] == current['end']):
             current['word'] += ' ' + next_entity['word']
             current['end'] = next_entity['end']
@@ -25,103 +24,181 @@ def merge_entities(entities):
     merged.append(current)
     return merged
 
-def ner(
-    text, labels: str, threshold: float, nested_ner: bool
-) -> Dict[str, Union[str, int, float]]:
-    labels = labels.split(",")
-    r = {
-        "text": text,
-        "entities": [
-            {
-                "entity": entity["label"],
-                "word": entity["text"],
-                "start": entity["start"],
-                "end": entity["end"],
-                "score": 0,
-            }
-            for entity in model.predict_entities(
-                text, labels, flat_ner=not nested_ner, threshold=threshold
-            )
-        ],
-    }
-    # r["entities"] =  merge_entities(r["entities"])
-    return r
+# Load the three models
+model_nuner = GLiNER.from_pretrained("numind/NuZero_token")
+model_pii   = GLiNER.from_pretrained("urchade/gliner_multi_pii-v1")
+model_med   = GLiNER.from_pretrained("urchade/gliner_medium-v2.1")
 
-with gr.Blocks(title="GLiNER-medium-v2.1") as demo:
-    gr.Markdown(
-        """
-        # GLiNER Testbed
+# Define NER functions for each model
+def ner_nuner(text, labels, threshold, nested_ner):
+    label_list = [lbl.strip() for lbl in labels.split(",")]
+    pred_entities = model_nuner.predict_entities(text, label_list, flat_ner=not nested_ner, threshold=threshold)
+    entities = [
+        {"entity": entity["label"], "word": entity["text"], "start": entity["start"], "end": entity["end"], "score": 0}
+        for entity in pred_entities
+    ]
+    merged_entities = merge_entities(entities)
+    return {"text": text, "entities": merged_entities}
 
-        GLiNER is a Named Entity Recognition (NER) model capable of identifying any entity type using a bidirectional transformer encoder (BERT-like). It provides a practical alternative to traditional NER models, which are limited to predefined entities, and Large Language Models (LLMs) that, despite their flexibility, are costly and large for resource-constrained scenarios. This model has the commercially permissive Apache 2.0 license.
+def ner_pii(text, labels, threshold, nested_ner):
+    label_list = [lbl.strip() for lbl in labels.split(",")]
+    pred_entities = model_pii.predict_entities(text, label_list, flat_ner=not nested_ner, threshold=threshold)
+    entities = [
+        {"entity": entity["label"], "word": entity["text"], "start": entity["start"], "end": entity["end"], "score": 0}
+        for entity in pred_entities
+    ]
+    return {"text": text, "entities": entities}
 
-        ## Links
+def ner_med(text, labels, threshold, nested_ner):
+    label_list = [lbl.strip() for lbl in labels.split(",")]
+    pred_entities = model_med.predict_entities(text, label_list, flat_ner=not nested_ner, threshold=threshold)
+    entities = [
+        {"entity": entity["label"], "word": entity["text"], "start": entity["start"], "end": entity["end"], "score": 0}
+        for entity in pred_entities
+    ]
+    return {"text": text, "entities": entities}
 
-        * Model: https://huggingface.co/urchade/gliner_medium-v2.1
-        * All GLiNER models: https://huggingface.co/models?library=gliner
-        * Paper: https://arxiv.org/abs/2311.08526
-        * Repository: https://github.com/urchade/GLiNER
-        """
-    )
+# Use the first example from the common examples for default values
+default_text, default_labels, default_threshold, default_nested = common_examples[0]
 
-    input_text = gr.Textbox(
-        value=examples[0][0], label="Text input", placeholder="Enter your text here"
-    )
-    with gr.Row() as row:
-        labels = gr.Textbox(
-            value=examples[0][1],
-            label="Labels",
-            placeholder="Enter your labels here (comma separated)",
-            scale=2,
-        )
-        threshold = gr.Slider(
-            0,
-            1,
-            value=0.3,
-            step=0.01,
-            label="Threshold",
-            info="Lower the threshold to increase how many entities get predicted.",
-            scale=1,
+# Build the combined Gradio app with three tabs
+with gr.Blocks(title="GLiNER NER Testbed") as demo:
+    gr.Markdown("# GLiNER NER Testbed")
+    with gr.Accordion("Detailed information on the approach", open=True):
+        gr.Markdown(
+            """
+            **GLiNER** is a state-of-the-art Named Entity Recognition (NER) system that leverages a BERT-like bidirectional transformer encoder to identify a wide range of entity types in text. Unlike conventional NER models that are restricted to fixed entity categories, GLiNER supports flexible, zero-shot extraction, making it ideal for diverse real-world applications. It also provides a resource-efficient alternative to large language models (LLMs) for scenarios where cost and speed are critical. Distributed under the Apache 2.0 license, GLiNER is commercially friendly and readily deployable.
+
+            **Useful Links**
+
+            - **Model:** [gliner_medium-v2.1](https://huggingface.co/urchade/gliner_medium-v2.1)
+            - **All GLiNER Models:** [Hugging Face GLiNER Models](https://huggingface.co/models?library=gliner)
+            - **Research Paper:** [arXiv:2311.08526](https://arxiv.org/abs/2311.08526)
+            - **Repository:** [GitHub - GLiNER](https://github.com/urchade/GLiNER)
+            """
         )
-        with gr.Column() as col:
-            nested_ner = gr.Checkbox(
-                value=examples[0][2],
-                label="Nested NER",
-                info="Allow for nested NER?",
-                scale=0,
-            )
-            merged_ent = gr.Checkbox(
-                #value=examples[0][3],
-                value=False,
-                label="Merged Entities",
-                info="Merge adjacent entities?",
-                scale=0,
+    
+    with gr.Tabs():
+        # Tab for GLiNER-medium
+        with gr.Tab("GLiNER-medium"):
+            gr.Markdown("## GLiNER-medium-v2.1")
+            with gr.Accordion("How to run this model locally", open=False):
+                gr.Markdown(
+                    """
+                    **Installation:**
+                    ```
+                    !pip install gliner
+                    ```
+                    **Usage:**
+                    Load the model with `GLiNER.from_pretrained("urchade/gliner_medium-v2.1")`
+                    and call `predict_entities` to perform zero-shot NER.
+                    """
+                )
+                gr.Code(
+                    '''from gliner import GLiNER
+model = GLiNER.from_pretrained("urchade/gliner_medium-v2.1")''',
+                    language="python",
+                )
+            input_text_med = gr.Textbox(value=default_text, label="Text input", placeholder="Enter your text here")
+            with gr.Row():
+                labels_med = gr.Textbox(value=default_labels, label="Labels", placeholder="Enter labels (comma separated)", scale=2)
+                threshold_med = gr.Slider(0, 1, value=default_threshold, step=0.01, label="Threshold", info="Lower threshold to increase predictions", scale=1)
+                nested_ner_med = gr.Checkbox(value=default_nested, label="Nested NER", info="Allow for nested NER?", scale=0)
+            output_med = gr.HighlightedText(label="Predicted Entities")
+            submit_btn_med = gr.Button("Submit")
+            gr.Examples(
+                common_examples,
+                fn=ner_med,
+                inputs=[input_text_med, labels_med, threshold_med, nested_ner_med],
+                outputs=output_med,
+                cache_examples=False,
             )
-    output = gr.HighlightedText(label="Predicted Entities")
-    submit_btn = gr.Button("Submit")
-    examples = gr.Examples(
-        examples,
-        fn=ner,
-        inputs=[input_text, labels, threshold, nested_ner],
-        outputs=output,
-        cache_examples=True,
-    )
+            input_text_med.submit(ner_med, inputs=[input_text_med, labels_med, threshold_med, nested_ner_med], outputs=output_med)
+            labels_med.submit(ner_med, inputs=[input_text_med, labels_med, threshold_med, nested_ner_med], outputs=output_med)
+            threshold_med.release(ner_med, inputs=[input_text_med, labels_med, threshold_med, nested_ner_med], outputs=output_med)
+            submit_btn_med.click(ner_med, inputs=[input_text_med, labels_med, threshold_med, nested_ner_med], outputs=output_med)
+            nested_ner_med.change(ner_med, inputs=[input_text_med, labels_med, threshold_med, nested_ner_med], outputs=output_med)
 
-    # Submitting
-    input_text.submit(
-        fn=ner, inputs=[input_text, labels, threshold, nested_ner], outputs=output
-    )
-    labels.submit(
-        fn=ner, inputs=[input_text, labels, threshold, nested_ner], outputs=output
-    )
-    threshold.release(
-        fn=ner, inputs=[input_text, labels, threshold, nested_ner], outputs=output
-    )
-    submit_btn.click(
-        fn=ner, inputs=[input_text, labels, threshold, nested_ner], outputs=output
-    )
-    nested_ner.change(
-        fn=ner, inputs=[input_text, labels, threshold, nested_ner], outputs=output
-    )
+        # Tab for GLiNER-PII
+        with gr.Tab("GLiNER-PII"):
+            gr.Markdown("## GLiNER-PII")
+            with gr.Accordion("How to run this model locally", open=False):
+                gr.Markdown(
+                    """
+                    **Installation:**
+                    ```
+                    !pip install gliner
+                    ```
+                    **Usage:**
+                    Load the model with `GLiNER.from_pretrained("urchade/gliner_multi_pii-v1")`
+                    and call `predict_entities` to extract PII.
+                    """
+                )
+                gr.Code(
+                    '''from gliner import GLiNER
+model = GLiNER.from_pretrained("urchade/gliner_multi_pii-v1")''',
+                    language="python",
+                )
+            input_text_pii = gr.Textbox(value=default_text, label="Text input", placeholder="Enter your text here")
+            with gr.Row():
+                labels_pii = gr.Textbox(value=default_labels, label="Labels", placeholder="Enter labels (comma separated)", scale=2)
+                threshold_pii = gr.Slider(0, 1, value=default_threshold, step=0.01, label="Threshold", info="Lower threshold to increase predictions", scale=1)
+                nested_ner_pii = gr.Checkbox(value=default_nested, label="Nested NER", info="Allow for nested NER?", scale=0)
+            output_pii = gr.HighlightedText(label="Predicted Entities")
+            submit_btn_pii = gr.Button("Submit")
+            gr.Examples(
+                common_examples,
+                fn=ner_pii,
+                inputs=[input_text_pii, labels_pii, threshold_pii, nested_ner_pii],
+                outputs=output_pii,
+                cache_examples=False,
+            )
+            input_text_pii.submit(ner_pii, inputs=[input_text_pii, labels_pii, threshold_pii, nested_ner_pii], outputs=output_pii)
+            labels_pii.submit(ner_pii, inputs=[input_text_pii, labels_pii, threshold_pii, nested_ner_pii], outputs=output_pii)
+            threshold_pii.release(ner_pii, inputs=[input_text_pii, labels_pii, threshold_pii, nested_ner_pii], outputs=output_pii)
+            submit_btn_pii.click(ner_pii, inputs=[input_text_pii, labels_pii, threshold_pii, nested_ner_pii], outputs=output_pii)
+            nested_ner_pii.change(ner_pii, inputs=[input_text_pii, labels_pii, threshold_pii, nested_ner_pii], outputs=output_pii)
+    
+        # Tab for NuNER Zero
+        with gr.Tab("NuNER Zero"):
+            gr.Markdown("## NuNER Zero")
+            with gr.Accordion("How to run this model locally", open=False):
+                gr.Markdown(
+                    """
+                    **Installation:**
+                    ```
+                    !pip install gliner
+                    ```
+                    **Usage:**
+                    Load the model with `GLiNER.from_pretrained("numind/NuZero_token")`
+                    and call `predict_entities` to perform zero-shot NER.
+                    """
+                )
+                gr.Code(
+                    '''from gliner import GLiNER
+model = GLiNER.from_pretrained("numind/NuZero_token")''',
+                    language="python",
+                )
+            input_text_nuner = gr.Textbox(value=default_text, label="Text input", placeholder="Enter your text here")
+            with gr.Row():
+                labels_nuner = gr.Textbox(value=default_labels, label="Labels", placeholder="Enter labels (comma separated)", scale=2)
+                threshold_nuner = gr.Slider(0, 1, value=default_threshold, step=0.01, label="Threshold", info="Lower threshold to increase predictions", scale=1)
+                nested_ner_nuner = gr.Checkbox(value=default_nested, label="Nested NER", info="Allow for nested NER?", scale=0)
+            output_nuner = gr.HighlightedText(label="Predicted Entities")
+            submit_btn_nuner = gr.Button("Submit")
+            gr.Examples(
+                common_examples,
+                fn=ner_nuner,
+                inputs=[input_text_nuner, labels_nuner, threshold_nuner, nested_ner_nuner],
+                outputs=output_nuner,
+                cache_examples=False,
+            )
+            input_text_nuner.submit(ner_nuner, inputs=[input_text_nuner, labels_nuner, threshold_nuner, nested_ner_nuner], outputs=output_nuner)
+            labels_nuner.submit(ner_nuner, inputs=[input_text_nuner, labels_nuner, threshold_nuner, nested_ner_nuner], outputs=output_nuner)
+            threshold_nuner.release(ner_nuner, inputs=[input_text_nuner, labels_nuner, threshold_nuner, nested_ner_nuner], outputs=output_nuner)
+            submit_btn_nuner.click(ner_nuner, inputs=[input_text_nuner, labels_nuner, threshold_nuner, nested_ner_nuner], outputs=output_nuner)
+            nested_ner_nuner.change(ner_nuner, inputs=[input_text_nuner, labels_nuner, threshold_nuner, nested_ner_nuner], outputs=output_nuner)
 
-demo.queue()
-demo.launch(debug=True)
+    # Enable queuing and launch the app
+    demo.queue()
+    demo.launch(debug=True)

app2.py

@@ -1,279 +0,0 @@
-import json
-from typing import Dict, Union, List
-from gliner import GLiNER
-import gradio as gr
-import os
-
-# Load available models
-MODELS = {
-    "GLiNER Medium v2.1": "urchade/gliner_medium-v2.1",
-    "NuNER Zero": "numind/NuZero_token",
-    "GLiNER Multi PII": "urchade/gliner_multi_pii-v1"
-}
-
-# Example datasets with descriptions
-EXAMPLE_SETS = {
-    "General NER": "examples.json",
-    "NuNER Zero": "examples-nuner.json",
-    "PII Detection": "examples-pii.json"
-}
-
-# Initialize models (will be loaded on demand)
-loaded_models = {}
-
-# Current examples
-current_examples = []
-
-def load_example_set(example_set_name):
-    """Load a set of examples from the specified file"""
-    try:
-        file_path = EXAMPLE_SETS[example_set_name]
-        with open(file_path, "r", encoding="utf-8") as f:
-            examples = json.load(f)
-        return examples
-    except (KeyError, FileNotFoundError, json.JSONDecodeError) as e:
-        print(f"Error loading example set {example_set_name}: {e}")
-        return []
-
-# Load default example set
-current_examples = load_example_set("General NER")
-
-def get_model(model_name):
-    """Load model if not already loaded"""
-    if model_name not in loaded_models:
-        model_path = MODELS[model_name]
-        loaded_models[model_name] = GLiNER.from_pretrained(model_path)
-    return loaded_models[model_name]
-
-def merge_entities(entities):
-    """Merge adjacent entities of the same type"""
-    if not entities:
-        return []
-    merged = []
-    current = entities[0]
-    for next_entity in entities[1:]:
-        if (next_entity['entity'] == current['entity'] and 
-            (next_entity['start'] == current['end'] + 1 or next_entity['start'] == current['end'])):
-            current['word'] += ' ' + next_entity['word']
-            current['end'] = next_entity['end']
-        else:
-            merged.append(current)
-            current = next_entity
-    merged.append(current)
-    return merged
-
-def ner(
-    text: str, 
-    labels: str, 
-    model_name: str,
-    threshold: float, 
-    nested_ner: bool,
-    merge_entities_toggle: bool
-) -> Dict[str, Union[str, List]]:
-    """Run named entity recognition with selected model and parameters"""
-    
-    # Get the selected model
-    model = get_model(model_name)
-    
-    # Split labels
-    label_list = [label.strip() for label in labels.split(",")]
-    
-    # Predict entities
-    entities = [
-        {
-            "entity": entity["label"],
-            "word": entity["text"],
-            "start": entity["start"],
-            "end": entity["end"],
-            "score": entity.get("score", 0),
-        }
-        for entity in model.predict_entities(
-            text, label_list, flat_ner=not nested_ner, threshold=threshold
-        )
-    ]
-    
-    # Merge entities if enabled
-    if merge_entities_toggle:
-        entities = merge_entities(entities)
-    
-    # Return results
-    return {
-        "text": text,
-        "entities": entities,
-    }
-
-def load_example(example_idx):
-    """Load a specific example by index from the current example set"""
-    if not current_examples or example_idx >= len(current_examples):
-        return "", "", 0.3, False, False
-    
-    example = current_examples[example_idx]
-    return example[0], example[1], example[2], example[3], False
-
-def switch_example_set(example_set_name):
-    """Switch to a different example set and update the interface"""
-    global current_examples
-    current_examples = load_example_set(example_set_name)
-    
-    # Return the first example from the new set
-    if current_examples:
-        example = current_examples[0]
-        # Return example text, labels, threshold, nested_ner, merge status, example names for dropdown
-        example_names = [f"Example {i+1}" for i in range(len(current_examples))]
-        return example[0], example[1], example[2], example[3], False, gr.Dropdown.update(choices=example_names, value="Example 1")
-    else:
-        return "", "", 0.3, False, False, gr.Dropdown.update(choices=[], value=None)
-
-with gr.Blocks(title="Unified NER Interface") as demo:
-    gr.Markdown(
-        """
-        # Unified Zero-shot Named Entity Recognition Interface
-        
-        This interface allows you to compare different zero-shot Named Entity Recognition models.
-        
-        ## Models Available:
-        - **GLiNER Medium v2.1**: The original GLiNER medium model
-        - **NuNER Zero**: A specialized token-based NER model
-        - **GLiNER Multi PII**: Fine-tuned for detecting personally identifiable information across multiple languages
-        
-        ## Features:
-        - Select different models
-        - Switch between example sets for different use cases
-        - Toggle nested entity recognition
-        - Toggle entity merging (combining adjacent entities of the same type)
-        - Select from various examples within each set
-        """
-    )
-    
-    with gr.Row():
-        model_dropdown = gr.Dropdown(
-            choices=list(MODELS.keys()),
-            value=list(MODELS.keys())[0],
-            label="Model",
-            info="Select the NER model to use"
-        )
-        example_set_dropdown = gr.Dropdown(
-            choices=list(EXAMPLE_SETS.keys()),
-            value="General NER",
-            label="Example Set",
-            info="Select a set of example texts"
-        )
-        
-    with gr.Row():
-        example_dropdown = gr.Dropdown(
-            choices=[f"Example {i+1}" for i in range(len(current_examples))],
-            value="Example 1",
-            label="Example",
-            info="Select a specific example text"
-        )
-    
-    input_text = gr.Textbox(
-        value=current_examples[0][0] if current_examples else "", 
-        label="Text input", 
-        placeholder="Enter your text here",
-        lines=5
-    )
-    
-    with gr.Row():
-        labels = gr.Textbox(
-            value=current_examples[0][1] if current_examples else "",
-            label="Entity Labels",
-            placeholder="Enter your labels here (comma separated)",
-            scale=2,
-        )
-        threshold = gr.Slider(
-            0,
-            1,
-            value=current_examples[0][2] if current_examples else 0.3,
-            step=0.01,
-            label="Confidence Threshold",
-            info="Lower the threshold to increase how many entities get predicted.",
-            scale=1,
-        )
-    
-    with gr.Row():
-        nested_ner = gr.Checkbox(
-            value=current_examples[0][3] if current_examples else False,
-            label="Nested NER",
-            info="Allow entities to be contained within other entities",
-        )
-        merge_entities_toggle = gr.Checkbox(
-            value=False,
-            label="Merge Adjacent Entities",
-            info="Combine adjacent entities of the same type into a single entity",
-        )
-    
-    output = gr.HighlightedText(label="Predicted Entities")
-    submit_btn = gr.Button("Submit")
-    
-    # Handling example set selection
-    example_set_dropdown.change(
-        fn=switch_example_set,
-        inputs=[example_set_dropdown],
-        outputs=[input_text, labels, threshold, nested_ner, merge_entities_toggle, example_dropdown]
-    )
-    
-    # Handling example selection within a set
-    example_dropdown.change(
-        fn=lambda idx: load_example(int(idx.split()[1]) - 1),
-        inputs=[example_dropdown],
-        outputs=[input_text, labels, threshold, nested_ner, merge_entities_toggle]
-    )
-    
-    # Add a model recommendation for the example set
-    def recommend_model(example_set_name):
-        """Recommend appropriate model based on example set"""
-        if example_set_name == "PII Detection":
-            return gr.Dropdown.update(value="GLiNER Multi PII")
-        elif example_set_name == "NuNER Zero":
-            return gr.Dropdown.update(value="NuNER Zero")
-        else:
-            return gr.Dropdown.update(value="GLiNER Medium v2.1")
-    
-    # Auto-suggest model when changing example set
-    example_set_dropdown.change(
-        fn=recommend_model,
-        inputs=[example_set_dropdown],
-        outputs=[model_dropdown]
-    )
-    
-    # Submitting
-    submit_btn.click(
-        fn=ner, 
-        inputs=[input_text, labels, model_dropdown, threshold, nested_ner, merge_entities_toggle], 
-        outputs=output
-    )
-    input_text.submit(
-        fn=ner, 
-        inputs=[input_text, labels, model_dropdown, threshold, nested_ner, merge_entities_toggle], 
-        outputs=output
-    )
-    
-    # Other interactions
-    model_dropdown.change(
-        fn=ner, 
-        inputs=[input_text, labels, model_dropdown, threshold, nested_ner, merge_entities_toggle], 
-        outputs=output
-    )
-    
-    threshold.release(
-        fn=ner, 
-        inputs=[input_text, labels, model_dropdown, threshold, nested_ner, merge_entities_toggle], 
-        outputs=output
-    )
-    
-    nested_ner.change(
-        fn=ner, 
-        inputs=[input_text, labels, model_dropdown, threshold, nested_ner, merge_entities_toggle], 
-        outputs=output
-    )
-    
-    merge_entities_toggle.change(
-        fn=ner, 
-        inputs=[input_text, labels, model_dropdown, threshold, nested_ner, merge_entities_toggle], 
-        outputs=output
-    )
-
-if __name__ == "__main__":
-    demo.queue()
-    demo.launch(debug=True)

examples-nuner.json

@@ -1,38 +0,0 @@
-[
-    [
-        "The Moon is Earth's only natural satellite. It orbits at an average distance of 384,400 km (238,900 mi), about 30 times the diameter of Earth. Over time Earth's gravity has caused tidal locking, causing the same side of the Moon to always face Earth. Because of this, the lunar day and the lunar month are the same length, at 29.5 Earth days. The Moon's gravitational pull – and to a lesser extent, the Sun's – are the main drivers of Earth's tides.",
-        "celestial body,quantity,physical concept",
-        0.3,
-        false
-    ],
-    [
-        "Their creation was inspired by the ancient Olympic Games, held in Olympia, Greece from the 8th century BC to the 4th century AD. Baron Pierre de Coubertin founded the International Olympic Committee (IOC) in 1894, leading to the first modern Games in Athens in 1896. The IOC is the governing body of the Olympic Movement, which encompasses all entities and individuals involved in the Olympic Games. The Olympic Charter defines their structure and authority.",
-        "location,date,person,event",
-        0.3,
-        true
-    ],
-    [
-        "Cells were discovered by Robert Hooke in 1665, who named them after their resemblance to cells inhabited by Christian monks in a monastery. Cell theory, developed in 1839 by Matthias Jakob Schleiden and Theodor Schwann, states that all organisms are composed of one or more cells, that cells are the fundamental unit of structure and function in all living organisms, and that all cells come from pre-existing cells.",
-        "biological concept,person,date",
-        0.3,
-        true
-    ],
-    [
-        "During the quarterly review, the CEO emphasized that the International Conference on 'Climate Change and Sustainable Practices for Emerging Economies in Southeast Asia', scheduled to be held next month in Paris, is a crucial platform for our team to present their groundbreaking research on renewable energy advancements. This conference, a collaboration between the United Nations Environmental Programme and various national governments, is recognized globally for facilitating critical discussions among world leaders, environmental scientists, and policymakers. The event aims to forge new partnerships and launch initiatives like 'Renewable Energy Deployment in Developing Regions: Challenges and Opportunities', a multi-year program seeking to address the unique energy needs of underdeveloped areas.",
-        "event,program",
-        0.3,
-        false
-    ],
-    [
-        "During the city council's strategic planning session, extensive references were made to the document titled 'Guidelines for Comprehensive Environmental Strategies in Urban Areas for the 21st Century: A Blueprint for Sustainable Urban Development'. This document serves as a foundational text for urban planners and local governments seeking to implement cutting-edge strategies for managing environmental impacts in rapidly growing metropolitan areas. It is complemented by the 'Metropolitan Environmental and Infrastructure Coordination Framework', which outlines specific policies and practices designed to enhance infrastructure resilience and sustainability in urban settings",
-        "document,framework",
-        0.3,
-        false
-    ],
-    [
-        "While preparing his thesis on the evolution of scientific thought, John delved into numerous sources, one of which was 'The Impact of Early Exploration on Modern Scientific Developments and Their Influence on Contemporary Scientific Thought: A Comprehensive Study of Geographical Discoveries and Their Lasting Impact on Modern Physics, Biology, and Sociopolitical Structures'. He found this text particularly enlightening, not only for its detailed analysis on how geographical discoveries influenced modern physics and biology but also for its exploration into the sociopolitical impacts these discoveries had on the scientific communities of the 17th and 18th centuries. In addition to this monumental work, he referenced 'Global Shifts in Technological Innovation During the Industrial Revolution', a book that examines the intersection of technology and industrial growth, and 'Philosophical Underpinnings of Modern Science', which offers insights into how Enlightenment philosophies molded scientific methods and inquiries.",
-        "book",
-        0.3,
-        false
-    ]
-]

examples-pii.json

@@ -1,92 +0,0 @@
-[
-    [
-        "Pierre Dubois, résident de Paris, a fondé sa propre entreprise, Le Petit Café, située au 15 Rue de la Paix. Son numéro d'entreprise est FR-987654321-1, et il utilise le compte bancaire 9876543210 pour les transactions.",
-        "person, organization, address, company registration number, bank account number",
-        0.5,
-        false
-    ],
-    [
-        "Leticia Ramírez, una habitante de Barcelona, tiene una cita médica programada en el Hospital General de Cataluña, situado en 10 Calle de los Ángeles. Su número de la seguridad social es ES-123456789-A y su grupo sanguíneo es AB+.",
-        "person, location, address, social security number, blood type",
-        0.5,
-        false
-    ],
-    [
-        "John Smith, from London, teaches mathematics at Royal Academy located at 25 King’s Road. His employee ID is UK-987654-321 and he has been working there since 2015.",
-        "person, profession, organization, address, employee ID number",
-        0.5,
-        false
-    ],
-    [
-        "In Frankfurt, Claudia Weber frequently visits her local bank branch, Deutsche Bank, at 48 Hauptstraße. Her account number is DE-1234567890123456, used primarily for her mortgage payments.",
-        "person, location, address, bank account number",
-        0.5,
-        false
-    ],
-    [
-        "Marta Rossi, residente a Roma, ha acquistato un appartamento al 123 Via Condotti. Il numero di registrazione della proprietà è IT-654321-2018 e il mutuo è gestito tramite la Banca d'Italia con numero di conto 3216549870.",
-        "person, address, property registration number, bank account number",
-        0.5,
-        false
-    ],
-    [
-        "Paulo Coelho, um turista do Brasil, fez um seguro de viagem com a empresa Seguros PT antes de sua viagem para Lisboa. O número da apólice é BR-987654321-123 e inclui cobertura médica.",
-        "person, nationality, company, insurance policy number, coverage",
-        0.5,
-        false
-    ],
-    [
-        "Julia Fischer, eine Kundin aus München, hat bei der BayWa AG, einem großen Anbieter von Baustoffen mit Sitz am 77 Industriestraße, einen Kredit aufgenommen. Die Kreditnummer lautet DE-12345678.",
-        "person, city, organization, address, loan number",
-        0.5,
-        false
-    ],
-    [
-        "Carlos Sánchez, profesor en la Universidad de Madrid, reside en el 5 Calle de Alcalá. Su número de identificación de profesor es ES-192837465 y tiene un doctorado en filosofía.",
-        "person, profession, address, teacher ID number, degree",
-        0.5,
-        false
-    ],
-    [
-        "Sophie Dupont, une journaliste française, travaille pour Le Monde, basé au 33 rue des Écoles à Paris. Son numéro d'identification de presse est FR-75649023.",
-        "person, profession, organization, address, press ID number",
-        0.5,
-        false
-    ],
-    [
-        "Manuel Oliveira, um agricultor em Porto, possui uma grande plantação de vinhas na Rua da Estrada, 120. O número de registro agrícola é PT-5678912345.",
-        "person, profession, address, agricultural registration number",
-        0.5,
-        false
-    ],
-    [
-        "Elisa Müller, eine Künstlerin aus Berlin, hat ihre neueste Skulptur im öffentlichen Park am Alexanderplatz ausgestellt. Ihre Künstlernummer lautet DE-112233445.",
-        "person, profession, location, artist ID number",
-        0.5,
-        false
-    ],
-    [
-        "Federico García, un jugador de fútbol de Sevilla, ha firmado un contrato de tres años con el club Real Betis. Su número de licencia deportiva es ES-9876543210.",
-        "person, profession, organization, sports license number",
-        0.5,
-        false
-    ],
-    [
-        "Sarah White, a London-based actress, will be performing in 'Hamlet' at the Globe Theatre located at 21 New Globe Walk. Her Equity membership number is UK-1234567.",
-        "person, profession, location, address, membership number",
-        0.5,
-        false
-    ],
-    [
-        "Ricardo Mello, engenheiro civil, trabalha na construção da nova barragem no Rio Douro, Portugal. Seu número de registro profissional é PT-987654321.",
-        "person, profession, project location, professional registration number",
-        0.5,
-        false
-    ],
-    [
-        "Giuseppe Conti, un cliente di Milano, ha fatto un acquisto presso il negozio La Rinascente situato in Piazza Duomo. Il numero della sua carta di credito è IT-4567891234567891.",
-        "person, location, address, credit card number",
-        0.5,
-        false
-    ]
-]

examples.json

@@ -1,12 +1,12 @@
 [
     [
-        "Libretto by Marius Petipa, based on the 1822 novella ``Trilby, ou Le Lutin d'Argail`` by Charles Nodier, first presented by the Ballet of the Moscow Imperial Bolshoi Theatre on January 25/February 6 (Julian/Gregorian calendar dates), 1870, in Moscow with Polina Karpakova as Trilby and Ludiia Geiten as Miranda and restaged by Petipa for the Imperial Ballet at the Imperial Bolshoi Kamenny Theatre on January 17\u201329, 1871 in St. Petersburg with Ad\u00e8le Grantzow as Trilby and Lev Ivanov as Count Leopold.",
+        "Libretto by Marius Petipa, based on the 1822 novella ``Trilby, ou Le Lutin d'Argail`` by Charles Nodier, first presented by the Ballet of the Moscow Imperial Bolshoi Theatre on January 25/February 6 (Julian/Gregorian calendar dates), 1870, in Moscow with Polina Karpakova as Trilby and Ludiia Geiten as Miranda and restaged by Petipa for the Imperial Ballet at the Imperial Bolshoi Kamenny Theatre on January 17–29, 1871 in St. Petersburg with Adèle Grantzow as Trilby and Lev Ivanov as Count Leopold.",
         "person, book, location, date, actor, character",
         0.3,
         true
     ],
     [
-        "\n* Data Scientist, Data Analyst, or Data Engineer with 1+ years of experience.\n* Experience with technologies such as Docker, Kubernetes, or Kubeflow\n* Machine Learning experience preferred\n* Experience with programming languages such as Python, C++, or SQL preferred\n* Experience with technologies such as Databricks, Qlik, TensorFlow, PyTorch, Python, Dash, Pandas, or NumPy preferred\n* BA or BS degree\n* Active Secret OR Active Top Secret or Active TS/SCI clearance\n",
+        "* Data Scientist, Data Analyst, or Data Engineer with 1+ years of experience.\n* Experience with technologies such as Docker, Kubernetes, or Kubeflow\n* Machine Learning experience preferred\n* Experience with programming languages such as Python, C++, or SQL preferred\n* Experience with technologies such as Databricks, Qlik, TensorFlow, PyTorch, Python, Dash, Pandas, or NumPy preferred\n* BA or BS degree\n* Active Secret OR Active Top Secret or Active TS/SCI clearance",
         "software package, programing language, software tool, degree, job title",
         0.3,
         false
@@ -66,7 +66,7 @@
         false
     ],
     [
-        "From November 29, 2011 to March 31, 2012, Karimloo returned to ``Les Mis\u00e9rables`` to play the lead role of Jean Valjean at The Queen's Theatre, London, for which he won the 2013 Theatregoers' Choice Award for Best Takeover in a Role.",
+        "From November 29, 2011 to March 31, 2012, Karimloo returned to ``Les Misérables`` to play the lead role of Jean Valjean at The Queen's Theatre, London, for which he won the 2013 Theatregoers' Choice Award for Best Takeover in a Role.",
         "person, actor, award, date, location",
         0.3,
         false
@@ -82,5 +82,131 @@
         "date, person, location, organization, event, flag",
         0.3,
         false
+    ],
+    [
+        "Pierre Dubois, résident de Paris, a fondé sa propre entreprise, Le Petit Café, située au 15 Rue de la Paix. Son numéro d'entreprise est FR-987654321-1, et il utilise le compte bancaire 9876543210 pour les transactions.",
+        "person, organization, address, company registration number, bank account number",
+        0.5,
+        false
+    ],
+    [
+        "Leticia Ramírez, una habitante de Barcelona, tiene una cita médica programada en el Hospital General de Cataluña, situado en 10 Calle de los Ángeles. Su número de la seguridad social es ES-123456789-A y su grupo sanguíneo es AB+.",
+        "person, location, address, social security number, blood type",
+        0.5,
+        false
+    ],
+    [
+        "John Smith, from London, teaches mathematics at Royal Academy located at 25 King’s Road. His employee ID is UK-987654-321 and he has been working there since 2015.",
+        "person, profession, organization, address, employee ID number",
+        0.5,
+        false
+    ],
+    [
+        "In Frankfurt, Claudia Weber frequently visits her local bank branch, Deutsche Bank, at 48 Hauptstraße. Her account number is DE-1234567890123456, used primarily for her mortgage payments.",
+        "person, location, address, bank account number",
+        0.5,
+        false
+    ],
+    [
+        "Marta Rossi, residente a Roma, ha acquistato un appartamento al 123 Via Condotti. Il numero di registrazione della proprietà è IT-654321-2018 e il mutuo è gestito tramite la Banca d'Italia con numero di conto 3216549870.",
+        "person, address, property registration number, bank account number",
+        0.5,
+        false
+    ],
+    [
+        "Paulo Coelho, um turista do Brasil, fez um seguro de viagem com a empresa Seguros PT antes de sua viagem para Lisboa. O número da apólice é BR-987654321-123 e inclui cobertura médica.",
+        "person, nationality, company, insurance policy number, coverage",
+        0.5,
+        false
+    ],
+    [
+        "Julia Fischer, eine Kundin aus München, hat bei der BayWa AG, einem großen Anbieter von Baustoffen mit Sitz am 77 Industriestraße, einen Kredit aufgenommen. Die Kreditnummer lautet DE-12345678.",
+        "person, city, organization, address, loan number",
+        0.5,
+        false
+    ],
+    [
+        "Carlos Sánchez, profesor en la Universidad de Madrid, reside en el 5 Calle de Alcalá. Su número de identificación de profesor es ES-192837465 y tiene un doctorado en filosofía.",
+        "person, profession, address, teacher ID number, degree",
+        0.5,
+        false
+    ],
+    [
+        "Sophie Dupont, une journaliste française, travaille pour Le Monde, basé au 33 rue des Écoles à Paris. Son numéro d'identification de presse est FR-75649023.",
+        "person, profession, organization, address, press ID number",
+        0.5,
+        false
+    ],
+    [
+        "Manuel Oliveira, um agricultor em Porto, possui uma grande plantação de vinhas na Rua da Estrada, 120. O número de registro agrícola é PT-5678912345.",
+        "person, profession, address, agricultural registration number",
+        0.5,
+        false
+    ],
+    [
+        "Elisa Müller, eine Künstlerin aus Berlin, hat ihre neueste Skulptur im öffentlichen Park am Alexanderplatz ausgestellt. Ihre Künstlernummer lautet DE-112233445.",
+        "person, profession, location, artist ID number",
+        0.5,
+        false
+    ],
+    [
+        "Federico García, un jugador de fútbol de Sevilla, ha firmado un contrato de tres años con el club Real Betis. Su número de licencia deportiva es ES-9876543210.",
+        "person, profession, organization, sports license number",
+        0.5,
+        false
+    ],
+    [
+        "Sarah White, a London-based actress, will be performing in 'Hamlet' at the Globe Theatre located at 21 New Globe Walk. Her Equity membership number is UK-1234567.",
+        "person, profession, location, address, membership number",
+        0.5,
+        false
+    ],
+    [
+        "Ricardo Mello, engenheiro civil, trabalha na construção da nova barragem no Rio Douro, Portugal. Seu número de registro profissional é PT-987654321.",
+        "person, profession, project location, professional registration number",
+        0.5,
+        false
+    ],
+    [
+        "Giuseppe Conti, un cliente di Milano, ha fatto un acquisto presso il negozio La Rinascente situato in Piazza Duomo. Il numero della sua carta di credito è IT-4567891234567891.",
+        "person, location, address, credit card number",
+        0.5,
+        false
+    ],
+    [
+        "The Moon is Earth's only natural satellite. It orbits at an average distance of 384,400 km (238,900 mi), about 30 times the diameter of Earth. Over time Earth's gravity has caused tidal locking, causing the same side of the Moon to always face Earth. Because of this, the lunar day and the lunar month are the same length, at 29.5 Earth days. The Moon's gravitational pull – and to a lesser extent, the Sun's – are the main drivers of Earth's tides.",
+        "celestial body,quantity,physical concept",
+        0.3,
+        false
+    ],
+    [
+        "Their creation was inspired by the ancient Olympic Games, held in Olympia, Greece from the 8th century BC to the 4th century AD. Baron Pierre de Coubertin founded the International Olympic Committee (IOC) in 1894, leading to the first modern Games in Athens in 1896. The IOC is the governing body of the Olympic Movement, which encompasses all entities and individuals involved in the Olympic Games. The Olympic Charter defines their structure and authority.",
+        "location,date,person,event",
+        0.3,
+        true
+    ],
+    [
+        "Cells were discovered by Robert Hooke in 1665, who named them after their resemblance to cells inhabited by Christian monks in a monastery. Cell theory, developed in 1839 by Matthias Jakob Schleiden and Theodor Schwann, states that all organisms are composed of one or more cells, that cells are the fundamental unit of structure and function in all living organisms, and that all cells come from pre-existing cells.",
+        "biological concept,person,date",
+        0.3,
+        true
+    ],
+    [
+        "During the quarterly review, the CEO emphasized that the International Conference on 'Climate Change and Sustainable Practices for Emerging Economies in Southeast Asia', scheduled to be held next month in Paris, is a crucial platform for our team to present their groundbreaking research on renewable energy advancements. This conference, a collaboration between the United Nations Environmental Programme and various national governments, is recognized globally for facilitating critical discussions among world leaders, environmental scientists, and policymakers. The event aims to forge new partnerships and launch initiatives like 'Renewable Energy Deployment in Developing Regions: Challenges and Opportunities', a multi-year program seeking to address the unique energy needs of underdeveloped areas.",
+        "event,program",
+        0.3,
+        false
+    ],
+    [
+        "During the city council's strategic planning session, extensive references were made to the document titled 'Guidelines for Comprehensive Environmental Strategies in Urban Areas for the 21st Century: A Blueprint for Sustainable Urban Development'. This document serves as a foundational text for urban planners and local governments seeking to implement cutting-edge strategies for managing environmental impacts in rapidly growing metropolitan areas. It is complemented by the 'Metropolitan Environmental and Infrastructure Coordination Framework', which outlines specific policies and practices designed to enhance infrastructure resilience and sustainability in urban settings",
+        "document,framework",
+        0.3,
+        false
+    ],
+    [
+        "While preparing his thesis on the evolution of scientific thought, John delved into numerous sources, one of which was 'The Impact of Early Exploration on Modern Scientific Developments and Their Influence on Contemporary Scientific Thought: A Comprehensive Study of Geographical Discoveries and Their Lasting Impact on Modern Physics, Biology, and Sociopolitical Structures'. He found this text particularly enlightening, not only for its detailed analysis on how geographical discoveries influenced modern physics and biology but also for its exploration into the sociopolitical impacts these discoveries had on the scientific communities of the 17th and 18th centuries. In addition to this monumental work, he referenced 'Global Shifts in Technological Innovation During the Industrial Revolution', a book that examines the intersection of technology and industrial growth, and 'Philosophical Underpinnings of Modern Science', which offers insights into how Enlightenment philosophies molded scientific methods and inquiries.",
+        "book",
+        0.3,
+        false
     ]
 ]