utils.py

"""This module contains utility functions for the project"""

import mmh3
from haystack import Document


def get_unique_docs(dataset, unique_docs: set):
    """Get unique documents from dataset

    Args:
    dataset: list of dictionaries

    Returns:
    docs: list of haystack.Document
    """
    docs = list()
    for doc in dataset:
        if doc["context"] is not None and doc["context_id"] not in unique_docs:
            unique_docs.add(doc["context_id"])
            document = Document(
                content=doc["context"],
                meta={
                    "title": doc["context_title"],
                    "context_id": doc["context_id"],
                    "url": doc["url"],
                    "source": "QASports",
                },
            )
            docs.append(document)
    return docs