|
--- |
|
license: mit |
|
datasets: |
|
- ZachW/GPT-BookSum |
|
language: |
|
- en |
|
metrics: |
|
- accuracy |
|
base_model: |
|
- FacebookAI/roberta-base |
|
pipeline_tag: zero-shot-classification |
|
tags: |
|
- pacing |
|
- concreteness |
|
- text-evalutaion |
|
--- |
|
# Pacing-Judge |
|
|
|
[\[project page\]](https://github.com/YichenZW/Pacing) |
|
|
|
## Overview |
|
|
|
This is the **concreteness evaluator** developed in the paper [Improving Pacing in Long-Form Story Planning](https://arxiv.org/abs/2311.04459) (EMNLP 2023). |
|
|
|
## Quick Start |
|
|
|
A simple usage: Input a pair of texts (text_ex_1, text_ex_2) with \<sep\> as the separator to the model. The output is whether the first or the second is more concrete. |
|
|
|
```python |
|
import torch.nn.functional as F |
|
from transformers import AutoModelForSequenceClassification, AutoTokenizer |
|
|
|
model_name = "ZachW/pacing-judge" |
|
model = AutoModelForSequenceClassification.from_pretrained(model_name) |
|
tokenizer = AutoTokenizer.from_pretrained(model_name) |
|
text_ex_1 = "The Duke then focused on securing his power and looking to future threats. The Duke eventually turned his attention to acquiring Tuscany but struggled." |
|
text_ex_2 = "Lord Bacon mentioned his book \"The History of Henry VII,\" in the conversation noting that King Charles had conquered Naples without resistance, implying that the conquest was like a dream." |
|
inputs = tokenizer(text_ex_1 + " <sep> " + text_ex_2, return_tensors="pt") |
|
outputs = model(**inputs) |
|
output = int(F.softmax(outputs.logits, dim=1)[:, 0].squeeze(-1).detach().cpu().numpy() > 0.5) |
|
print(f"Output Binary = {output}") |
|
if output: |
|
print("The second text is more concrete.") |
|
else: |
|
print("The first text is more concrete.") |
|
``` |
|
|
|
## Usage |
|
|
|
We have designed this Ranker, which enables fair pairwise comparison (independent of sequence order) and ranking among candidates. We **recommend** using our model via the Ranker. |
|
|
|
```python |
|
import torch.nn.functional as F |
|
from transformers import AutoModelForSequenceClassification, AutoTokenizer |
|
|
|
class Ranker: |
|
def __init__(self): |
|
print(f"*** Loading Model from Huggingface ***") |
|
model_name = "ZachW/pacing-judge" |
|
self.model = AutoModelForSequenceClassification.from_pretrained(model_name) |
|
self.tokenizer = AutoTokenizer.from_pretrained(model_name) |
|
|
|
def compare(self, t1, t2): |
|
text_pair = [t1 + ' <sep> ' + t2, t2 + ' <sep> ' + t1] |
|
pair_dataset = self.tokenizer(text_pair, padding=True, truncation=True, return_tensors="pt") |
|
score = self.run_model(pair_dataset) |
|
if score < 0.5: |
|
return 0 # first is more concrete |
|
else: |
|
return 1 # second is more concrete |
|
|
|
def compare_logits(self, t1, t2): |
|
text_pair = [t1 + ' <sep> ' + t2, t2 + ' <sep> ' + t1] |
|
pair_dataset = self.tokenizer(text_pair, padding=True, truncation=True, return_tensors="pt") |
|
score = self.run_model(pair_dataset) |
|
return score |
|
|
|
def run_model(self, dataset): |
|
outputs = self.model(**dataset) |
|
scores = F.softmax(outputs.logits, dim=1)[:, 0].squeeze(-1).detach().cpu().numpy() |
|
aver_score = (scores[0] + (1 - scores[1]))/2 |
|
return aver_score |
|
|
|
def rank(self, texts_list): # input a list of texts |
|
def quicksort(arr): |
|
if len(arr) <= 1: |
|
return arr |
|
else: |
|
pivot = arr[0] |
|
less = [] |
|
greater = [] |
|
for t in arr[1:]: |
|
cmp = self.compare(pivot, t) |
|
if cmp == 0: |
|
less.append(t) |
|
elif cmp == 1: |
|
greater.append(t) |
|
return quicksort(greater) + [pivot] + quicksort(less) |
|
return quicksort(texts_list) |
|
# most concrete -> lest concrete |
|
|
|
def rank_idx(self, texts_list): # input a list of texts |
|
def quicksort(arr): |
|
if len(arr) <= 1: |
|
return arr |
|
else: |
|
pivot = arr[0] |
|
less = [] |
|
greater = [] |
|
for t in arr[1:]: |
|
cmp = self.compare(texts_list[pivot], texts_list[t]) |
|
if cmp == 0: |
|
less.append(t) |
|
elif cmp == 1: |
|
greater.append(t) |
|
return quicksort(greater) + [pivot] + quicksort(less) |
|
return quicksort(list(range(len(texts_list)))) |
|
|
|
def rank_idx_conpletely(self, texts_list): |
|
n = len(texts_list) |
|
texts_idx = list(range(n)) |
|
scores = [[0] * n for _ in range(n)] |
|
self_score = [0] * n |
|
for i in texts_idx: |
|
scores[i][i] = self.compare_logits(texts_list[i], texts_list[i]) |
|
self_score[i] = scores[i][i] |
|
for j in texts_idx: |
|
if j < i: |
|
scores[i][j] = 1 - scores[j][i] |
|
continue |
|
if j == i: |
|
continue |
|
scores[i][j] = self.compare_logits(texts_list[i], texts_list[j]) |
|
# average score is, smaller is more concrete |
|
average_score = [ sum(s)/len(s) for s in scores] |
|
output_score = [ a + 0.5 - s for a, s in zip(average_score, self_score)] |
|
sorted_indices = sorted(range(len(output_score)), key=lambda x: output_score[x]) |
|
return sorted_indices |
|
|
|
def rank_idx_conpletely_wlogits(self, texts_list, logger=None): |
|
n = len(texts_list) |
|
texts_idx = list(range(n)) |
|
scores = [[0] * n for _ in range(n)] |
|
self_score = [0] * n |
|
for i in texts_idx: |
|
scores[i][i] = self.compare_logits(texts_list[i], texts_list[i]) |
|
self_score[i] = scores[i][i] |
|
for j in texts_idx: |
|
if j < i: |
|
scores[i][j] = 1 - scores[j][i] |
|
continue |
|
if j == i: |
|
continue |
|
scores[i][j] = self.compare_logits(texts_list[i], texts_list[j]) |
|
# average score is, smaller is more concrete |
|
average_score = [ sum(s)/len(s) for s in scores] |
|
output_score = [ a + 0.5 - s for a, s in zip(average_score, self_score)] |
|
sorted_indices = sorted(range(len(output_score)), key=lambda x: output_score[x]) |
|
return sorted_indices, output_score |
|
|
|
def compare_w_neighbors(self, t, cand): |
|
score = 0.0 |
|
for c in cand: |
|
score += self.compare_logits(t, c) |
|
score /= len(cand) |
|
return score |
|
``` |
|
|
|
```python |
|
text_ex_1 = "The Duke then focused on securing his power and looking to future threats. The Duke eventually turned his attention to acquiring Tuscany but struggled." |
|
text_ex_2 = "Lord Bacon mentioned his book \"The History of Henry VII,\" in the conversation noting that King Charles had conquered Naples without resistance, implying that the conquest was like a dream." |
|
|
|
ranker = Ranker() |
|
output = ranker.compare(text_ex_1, text_ex_2) # it is equvilant to (text_ex_2, text_ex_1) |
|
print(f"Output Binary = {output}") |
|
if output: |
|
print("The second text is more concrete.") |
|
else: |
|
print("The first text is more concrete.") |
|
|
|
output_logits = ranker.compare_logits(text_ex_1, text_ex_2) |
|
print(f"Output Logits = {output_logits:.4f}") |
|
``` |
|
|
|
**For more details on the evaluator usage (e.g., pacing planning and control in generation) and training process, please refer to our [paper](https://arxiv.org/abs/2311.04459)!** |
|
|
