Commit
·
31531b7
1
Parent(s):
0855882
Update README.md
Browse files
README.md
CHANGED
|
@@ -54,7 +54,6 @@ from transformers import (
|
|
| 54 |
AutoModelForSeq2SeqLM,
|
| 55 |
AutoTokenizer,
|
| 56 |
)
|
| 57 |
-
import numpy as np
|
| 58 |
|
| 59 |
|
| 60 |
class KeyphraseGenerationPipeline(Text2TextGenerationPipeline):
|
|
@@ -71,22 +70,24 @@ class KeyphraseGenerationPipeline(Text2TextGenerationPipeline):
|
|
| 71 |
results = super().postprocess(
|
| 72 |
model_outputs=model_outputs
|
| 73 |
)
|
| 74 |
-
return [[keyphrase.strip() for keyphrase in result.get("generated_text").split(self.keyphrase_sep_token)] for result in results]
|
|
|
|
| 75 |
```
|
| 76 |
|
| 77 |
```python
|
| 78 |
# Load pipeline
|
| 79 |
-
model_name = "
|
| 80 |
generator = KeyphraseGenerationPipeline(model=model_name)
|
| 81 |
|
| 82 |
```python
|
| 83 |
text = """
|
| 84 |
Keyphrase extraction is a technique in text analysis where you extract the important keyphrases from a text.
|
| 85 |
Since this is a time-consuming process, Artificial Intelligence is used to automate it.
|
| 86 |
-
Currently, classical machine learning methods, that use statistics and linguistics,
|
| 87 |
-
The fact that these methods have been widely used in the community
|
| 88 |
-
|
| 89 |
-
|
|
|
|
| 90 |
""".replace(
|
| 91 |
"\n", ""
|
| 92 |
)
|
|
@@ -99,7 +100,7 @@ print(keyphrases)
|
|
| 99 |
|
| 100 |
```
|
| 101 |
# Output
|
| 102 |
-
[['keyphrase extraction', 'text analysis', 'artificial intelligence'
|
| 103 |
```
|
| 104 |
|
| 105 |
## 📚 Training Dataset
|
|
@@ -200,4 +201,4 @@ Abstractive keyphrases
|
|
| 200 |
For more information on the evaluation process, you can take a look at the keyphrase extraction evaluation notebook.
|
| 201 |
|
| 202 |
## 🚨 Issues
|
| 203 |
-
Please feel free to
|
|
|
|
| 54 |
AutoModelForSeq2SeqLM,
|
| 55 |
AutoTokenizer,
|
| 56 |
)
|
|
|
|
| 57 |
|
| 58 |
|
| 59 |
class KeyphraseGenerationPipeline(Text2TextGenerationPipeline):
|
|
|
|
| 70 |
results = super().postprocess(
|
| 71 |
model_outputs=model_outputs
|
| 72 |
)
|
| 73 |
+
return [[keyphrase.strip() for keyphrase in result.get("generated_text").split(self.keyphrase_sep_token) if keyphrase != ""] for result in results]
|
| 74 |
+
|
| 75 |
```
|
| 76 |
|
| 77 |
```python
|
| 78 |
# Load pipeline
|
| 79 |
+
model_name = "ml6team/keyphrase-generation-t5-small-openkp"
|
| 80 |
generator = KeyphraseGenerationPipeline(model=model_name)
|
| 81 |
|
| 82 |
```python
|
| 83 |
text = """
|
| 84 |
Keyphrase extraction is a technique in text analysis where you extract the important keyphrases from a text.
|
| 85 |
Since this is a time-consuming process, Artificial Intelligence is used to automate it.
|
| 86 |
+
Currently, classical machine learning methods, that use statistics and linguistics,
|
| 87 |
+
are widely used for the extraction process. The fact that these methods have been widely used in the community
|
| 88 |
+
has the advantage that there are many easy-to-use libraries. Now with the recent innovations in NLP,
|
| 89 |
+
transformers can be used to improve keyphrase extraction. Transformers also focus on the semantics
|
| 90 |
+
and context of a document, which is quite an improvement.
|
| 91 |
""".replace(
|
| 92 |
"\n", ""
|
| 93 |
)
|
|
|
|
| 100 |
|
| 101 |
```
|
| 102 |
# Output
|
| 103 |
+
[['keyphrase extraction', 'text analysis', 'artificial intelligence']]
|
| 104 |
```
|
| 105 |
|
| 106 |
## 📚 Training Dataset
|
|
|
|
| 201 |
For more information on the evaluation process, you can take a look at the keyphrase extraction evaluation notebook.
|
| 202 |
|
| 203 |
## 🚨 Issues
|
| 204 |
+
Please feel free to start discussions in the Community Tab.
|