Spaces:
Running
Running
File size: 1,327 Bytes
8066b54 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
from typing import List
import PyPDF2
import io
class CharacterTextSplitter:
def __init__(self, chunk_size=1000, chunk_overlap=200):
self.chunk_size = chunk_size
self.chunk_overlap = chunk_overlap
def split_texts(self, documents: List[str]) -> List[str]:
texts = []
for doc in documents:
if not doc.strip():
continue
# Split the text into chunks
start = 0
while start < len(doc):
end = start + self.chunk_size
chunk = doc[start:end]
texts.append(chunk)
start = end - self.chunk_overlap
return texts
class TextFileLoader:
def __init__(self, file_path: str):
self.file_path = file_path
def load_documents(self) -> List[str]:
with open(self.file_path, 'r', encoding='utf-8') as file:
return [file.read()]
class PDFLoader:
def __init__(self, file_path: str):
self.file_path = file_path
def load_documents(self) -> List[str]:
with open(self.file_path, 'rb') as file:
pdf_reader = PyPDF2.PdfReader(file)
text = ""
for page in pdf_reader.pages:
text += page.extract_text() + "\n"
return [text] |