Duplicate from eson/bert-perplexity-debug

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+flagged/
+.Python
+build/
+develop-eggs/
+dist/
+eggs/
+.eggs/
+.idea/

README.md

@@ -0,0 +1,13 @@
+---
+title: Bert Perplexity
+emoji: 💩
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: 3.18.0
+app_file: app.py
+pinned: false
+duplicated_from: eson/bert-perplexity-debug
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,58 @@
+# coding=utf-8
+# author: xusong
+# time: 2022/8/23 16:06
+
+from perplexity import PerplexityPipeline
+from transformers import BertTokenizer, BertForMaskedLM
+import gradio as gr
+import time
+
+en_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+en_model = BertForMaskedLM.from_pretrained("bert-base-uncased")
+en_pipeline = PerplexityPipeline(model=en_model, tokenizer=en_tokenizer)
+
+zh_tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
+zh_model = BertForMaskedLM.from_pretrained("bert-base-chinese")
+zh_pipeline = PerplexityPipeline(model=zh_model, tokenizer=zh_tokenizer)
+
+
+def ppl(model_version, text):
+    print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), model_version, text)
+    if model_version == "bert-base-uncased":
+        result = en_pipeline(text)
+    else:
+        result = zh_pipeline(text)
+    return result["ppl"], result
+
+
+examples = [
+    ["bert-base-uncased", "New York City is located in the northeastern United States."],
+    ["bert-base-uncased", "New York City is located in the western United States."],
+    ["bert-base-chinese", "少先队员因该为老人让坐"],
+]
+
+css = "#json-container {height:: 400px; overflow: auto !important}"
+
+corr_iface = gr.Interface(
+    fn=ppl,
+    inputs=[
+        # gr.Dropdown(["bert-base-uncased", "bert-base-chinese"], value="bert-base-uncased"), # TODO 调整大小和位置
+        gr.Radio(
+            ["bert-base-uncased", "bert-base-chinese"],
+            value="bert-base-uncased"
+        ),
+        gr.Textbox(
+            value="New York City is located in the northeastern United States.",
+            label="input text"
+        )],
+    outputs=[
+        gr.Textbox(label="Perplexity"),
+        gr.JSON(label="Tokens", elem_id="json-container")],
+    examples=examples,
+    title="BERT as Language Model",
+    description='',
+    css=css
+)
+
+if __name__ == "__main__":
+    corr_iface.launch()

perplexity.py

@@ -0,0 +1,57 @@
+# coding=utf-8
+# author: xusong
+# time: 2022/8/22 12:06
+
+import numpy as np
+import torch
+from transformers import FillMaskPipeline
+
+
+class PerplexityPipeline(FillMaskPipeline):
+
+    def create_sequential_mask(self, input_data, mask_count=1):
+        _, seq_length = input_data["input_ids"].shape
+        mask_count = seq_length - 2
+
+        input_ids = input_data["input_ids"]
+
+        new_input_ids = torch.repeat_interleave(input_data["input_ids"], repeats=mask_count, dim=0)
+        token_type_ids = torch.repeat_interleave(input_data["token_type_ids"], repeats=mask_count, dim=0)
+        attention_mask = torch.repeat_interleave(input_data["attention_mask"], repeats=mask_count, dim=0)
+        masked_lm_labels = []
+        masked_lm_positions = list(range(1, mask_count + 1))
+        for i in masked_lm_positions:
+            new_input_ids[i - 1][i] = self.tokenizer.mask_token_id
+            masked_lm_labels.append(input_ids[0][i].item())
+        new_data = {"input_ids": new_input_ids, "token_type_ids": token_type_ids, "attention_mask": attention_mask}
+        return new_data, masked_lm_positions, masked_lm_labels
+
+    def __call__(self, input_text, *args, **kwargs):
+        """
+        Compute perplexity for given sentence.
+        """
+        if not isinstance(input_text, str):
+            return None
+        # 1. create sequential mask
+        model_inputs = self.tokenizer(input_text, return_tensors='pt')
+        new_data, masked_lm_positions, masked_lm_labels = self.create_sequential_mask(model_inputs.data)
+        model_inputs.data = new_data
+        labels = torch.tensor(masked_lm_labels)
+
+        # 2. predict
+        model_outputs = self.model(**model_inputs)
+
+        # 3. compute perplexity
+        sentence = {}
+        tokens = []
+        for i in range(len(labels)):
+            model_outputs_i = {}
+            model_outputs_i["input_ids"] = model_inputs["input_ids"][i:i + 1]
+            model_outputs_i["logits"] = model_outputs["logits"][i:i + 1]
+            outputs = self.postprocess(model_outputs_i, target_ids=labels[i:i + 1])
+            print(outputs)
+            tokens.append({"token": outputs[0]["token_str"],
+                           "prob": outputs[0]["score"]})
+        sentence["tokens"] = tokens
+        sentence["ppl"] = float(np.exp(- sum(np.log(token["prob"]) for token in tokens) / len(tokens)))
+        return sentence

requirements.txt

@@ -0,0 +1,2 @@
+transformers>=4.21.1
+torch