import re
import logging
from haystack.document_stores import ElasticsearchDocumentStore
from haystack.utils import launch_es,print_answers
from haystack.nodes import FARMReader,TransformersReader,BM25Retriever
from haystack.pipelines import ExtractiveQAPipeline
from haystack.nodes import TextConverter,PDFToTextConverter,PreProcessor
from haystack.utils import convert_files_to_docs, fetch_archive_from_http
from Reader import PdfReader,ExtractedText
launch_es() # Launches an Elasticsearch instance on your local machine
# Install the latest release of Haystack in your own environment
#! pip install farm-haystack
"""Install the latest main of Haystack"""
# !pip install --upgrade pip
# !pip install git+https://github.com/deepset-ai/haystack.git#egg=farm-haystack[colab,ocr]
# # For Colab/linux based machines
# !wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-linux-4.04.tar.gz
# !tar -xvf xpdf-tools-linux-4.04.tar.gz && sudo cp xpdf-tools-linux-4.04/bin64/pdftotext /usr/local/bin
# For Macos machines
# !wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-mac-4.03.tar.gz
# !tar -xvf xpdf-tools-mac-4.03.tar.gz && sudo cp xpdf-tools-mac-4.03/bin64/pdftotext /usr/local/bin
"Run this script from the root of the project"
# # In Colab / No Docker environments: Start Elasticsearch from source
# ! wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.2-linux-x86_64.tar.gz -q
# ! tar -xzf elasticsearch-7.9.2-linux-x86_64.tar.gz
# ! chown -R daemon:daemon elasticsearch-7.9.2
# import os
# from subprocess import Popen, PIPE, STDOUT
# es_server = Popen(
# ["elasticsearch-7.9.2/bin/elasticsearch"], stdout=PIPE, stderr=STDOUT, preexec_fn=lambda: os.setuid(1) # as daemon
# )
# # wait until ES has started
# ! sleep 30
logging.basicConfig(format="%(levelname)s - %(name)s - %(message)s", level=logging.WARNING)
logging.getLogger("haystack").setLevel(logging.INFO)
class Connection:
def __init__(self,host="localhost",username="",password="",index="document"):
"""
host: Elasticsearch host. If no host is provided, the default host "localhost" is used.
port: Elasticsearch port. If no port is provided, the default port 9200 is used.
username: Elasticsearch username. If no username is provided, no username is used.
password: Elasticsearch password. If no password is provided, no password is used.
index: Elasticsearch index. If no index is provided, the default index "document" is used.
"""
self.host=host
self.username=username
self.password=password
self.index=index
def get_connection(self):
document_store=ElasticsearchDocumentStore(host=self.host,username=self.username,password=self.password,index=self.index)
return document_store
class QAHaystack:
def __init__(self, filename):
self.filename=filename
def preprocessing(self,data):
"""
This function is used to preprocess the data. Its a simple function which removes the special characters and converts the data to lower case.
"""
converter = TextConverter(remove_numeric_tables=True, valid_languages=["en"])
doc_txt = converter.convert(file_path=ExtractedText(self.filename,'data.txt').save(4,6), meta=None)[0]
converter = PDFToTextConverter(remove_numeric_tables=True, valid_languages=["en"])
doc_pdf = converter.convert(file_path="data/tutorial8/manibook.pdf", meta=None)[0]
preprocess_text=data.lower() # lowercase
preprocess_text = re.sub(r'\s+', ' ', preprocess_text) # remove extra spaces
return preprocess_text
def convert_to_document(self,data):
"""
Write the data to a text file. This is required since the haystack library requires the data to be in a text file so that it can then be converted to a document.
"""
data=self.preprocessing(data)
with open(self.filename,'w') as f:
f.write(data)
"""
Read the data from the text file.
"""
data=self.preprocessing(data)
with open(self.filename,'r') as f:
data=f.read()
data=data.split("\n")
"""
DocumentStores expect Documents in dictionary form, like that below. They are loaded using the DocumentStore.write_documents()
dicts=[
{
'content': DOCUMENT_TEXT_HERE,
'meta':{'name': DOCUMENT_NAME,...}
},...
]
(Optionally: you can also add more key-value-pairs here, that will be indexed as fields in Elasticsearch and can be accessed later for filtering or shown in the responses of the Pipeline)
"""
data_json=[{
'content':paragraph,
'meta':{
'name':self.filename
}
} for paragraph in data
]
document_store=Connection().get_connection()
document_store.write_documents(data_json)
return document_store
class Pipeline:
def __init__(self,filename,retriever=BM25Retriever,reader=FARMReader):
self.reader=reader
self.retriever=retriever
self.filename=filename
def get_prediction(self,data,query):
"""
Retrievers help narrowing down the scope for the Reader to smaller units of text where a given question could be answered. They use some simple but fast algorithm.
Here: We use Elasticsearch's default BM25 algorithm . I'll check out the other retrievers as well.
"""
retriever=self.retriever(document_store=QAHaystack(self.filename).convert_to_document(data))
"""
Readers scan the texts returned by retrievers in detail and extract k best answers. They are based on powerful, but slower deep learning models.Haystack currently supports Readers based on the frameworks FARM and Transformers.
"""
reader = self.reader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=True)
"""
With a Haystack Pipeline we can stick together your building blocks to a search pipeline. Under the hood, Pipelines are Directed Acyclic Graphs (DAGs) that you can easily customize for our own use cases. To speed things up, Haystack also comes with a few predefined Pipelines. One of them is the ExtractiveQAPipeline that combines a retriever and a reader to answer our questions.
"""
pipe = ExtractiveQAPipeline(reader, retriever)
"""
This function is used to get the prediction from the pipeline.
"""
prediction = pipe.run(query=query, params={"Retriever":{"top_k":10}, "Reader":{"top_k":5}})
return prediction