Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -26,7 +26,7 @@ We will load our environment variables here.
|
|
26 |
"""
|
27 |
HF_LLM_ENDPOINT = os.environ["HF_LLM_ENDPOINT"]
|
28 |
HF_EMBED_ENDPOINT = os.environ["HF_EMBED_ENDPOINT"]
|
29 |
-
HF_TOKEN = os.environ["
|
30 |
|
31 |
# ---- GLOBAL DECLARATIONS ---- #
|
32 |
|
@@ -37,7 +37,7 @@ HF_TOKEN = os.environ["HF_TOKEN"]
|
|
37 |
3. Load HuggingFace Embeddings (remember to use the URL we set above)
|
38 |
4. Index Files if they do not exist, otherwise load the vectorstore
|
39 |
"""
|
40 |
-
document_loader = TextLoader("
|
41 |
documents = document_loader.load()
|
42 |
|
43 |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=30)
|
@@ -49,9 +49,9 @@ hf_embeddings = HuggingFaceEndpointEmbeddings(
|
|
49 |
huggingfacehub_api_token=HF_TOKEN,
|
50 |
)
|
51 |
|
52 |
-
if os.path.exists("
|
53 |
vectorstore = FAISS.load_local(
|
54 |
-
"
|
55 |
hf_embeddings,
|
56 |
allow_dangerous_deserialization=True, # this is necessary to load the vectorstore from disk as it's stored as a `.pkl` file.
|
57 |
)
|
@@ -59,7 +59,7 @@ if os.path.exists("./data/vectorstore"):
|
|
59 |
print("Loaded Vectorstore")
|
60 |
else:
|
61 |
print("Indexing Files")
|
62 |
-
os.makedirs("
|
63 |
for i in range(0, len(split_documents), 32):
|
64 |
if i == 0:
|
65 |
vectorstore = FAISS.from_documents(
|
@@ -67,7 +67,7 @@ else:
|
|
67 |
)
|
68 |
continue
|
69 |
vectorstore.add_documents(split_documents[i : i + 32])
|
70 |
-
vectorstore.save_local("
|
71 |
|
72 |
hf_retriever = vectorstore.as_retriever()
|
73 |
|
|
|
26 |
"""
|
27 |
HF_LLM_ENDPOINT = os.environ["HF_LLM_ENDPOINT"]
|
28 |
HF_EMBED_ENDPOINT = os.environ["HF_EMBED_ENDPOINT"]
|
29 |
+
HF_TOKEN = os.environ["HF_API_KEY"]
|
30 |
|
31 |
# ---- GLOBAL DECLARATIONS ---- #
|
32 |
|
|
|
37 |
3. Load HuggingFace Embeddings (remember to use the URL we set above)
|
38 |
4. Index Files if they do not exist, otherwise load the vectorstore
|
39 |
"""
|
40 |
+
document_loader = TextLoader("paul_graham_essays.txt")
|
41 |
documents = document_loader.load()
|
42 |
|
43 |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=30)
|
|
|
49 |
huggingfacehub_api_token=HF_TOKEN,
|
50 |
)
|
51 |
|
52 |
+
if os.path.exists("vectorstore"):
|
53 |
vectorstore = FAISS.load_local(
|
54 |
+
"vectorstore",
|
55 |
hf_embeddings,
|
56 |
allow_dangerous_deserialization=True, # this is necessary to load the vectorstore from disk as it's stored as a `.pkl` file.
|
57 |
)
|
|
|
59 |
print("Loaded Vectorstore")
|
60 |
else:
|
61 |
print("Indexing Files")
|
62 |
+
os.makedirs("vectorstore", exist_ok=True)
|
63 |
for i in range(0, len(split_documents), 32):
|
64 |
if i == 0:
|
65 |
vectorstore = FAISS.from_documents(
|
|
|
67 |
)
|
68 |
continue
|
69 |
vectorstore.add_documents(split_documents[i : i + 32])
|
70 |
+
vectorstore.save_local("vectorstore")
|
71 |
|
72 |
hf_retriever = vectorstore.as_retriever()
|
73 |
|