Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -1,17 +1,19 @@
|
|
1 |
-
# config.py
|
2 |
import os
|
|
|
|
|
|
|
|
|
|
|
|
|
3 |
|
|
|
4 |
class Config:
|
5 |
OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
|
6 |
MODEL_NAME = "gpt-3.5-turbo"
|
7 |
EMBEDDING_MODEL = "text-embedding-3-small"
|
8 |
CHUNK_SIZE = 256
|
9 |
|
10 |
-
#
|
11 |
-
from llama_index.core import SimpleDirectoryReader, VectorStoreIndex, StorageContext
|
12 |
-
from llama_index.core.node_parser import SentenceSplitter
|
13 |
-
import streamlit as st
|
14 |
-
|
15 |
class DocumentProcessor:
|
16 |
def __init__(self):
|
17 |
self.splitter = SentenceSplitter(chunk_size=Config.CHUNK_SIZE)
|
@@ -31,10 +33,7 @@ class DocumentProcessor:
|
|
31 |
storage_context.docstore.add_documents(nodes)
|
32 |
return VectorStoreIndex(nodes=nodes, storage_context=storage_context), nodes
|
33 |
|
34 |
-
#
|
35 |
-
from llama_index.retrievers.bm25 import BM25Retriever
|
36 |
-
from llama_index.core.retrievers import BaseRetriever
|
37 |
-
|
38 |
class HybridRetriever(BaseRetriever):
|
39 |
def __init__(self, vector_retriever, bm25_retriever):
|
40 |
self.vector_retriever = vector_retriever
|
@@ -51,9 +50,7 @@ class HybridRetriever(BaseRetriever):
|
|
51 |
node_ids.add(n.node.node_id)
|
52 |
return all_nodes
|
53 |
|
54 |
-
#
|
55 |
-
import openai
|
56 |
-
|
57 |
class LLMService:
|
58 |
def __init__(self, model_name):
|
59 |
self.model_name = model_name
|
@@ -87,13 +84,7 @@ class LLMService:
|
|
87 |
)
|
88 |
return response.choices[0].message.content
|
89 |
|
90 |
-
#
|
91 |
-
import streamlit as st
|
92 |
-
from config import Config
|
93 |
-
from document_processor import DocumentProcessor
|
94 |
-
from retriever import HybridRetriever
|
95 |
-
from llm_service import LLMService
|
96 |
-
|
97 |
class PromptOptimizationApp:
|
98 |
def __init__(self):
|
99 |
self.doc_processor = DocumentProcessor()
|
|
|
|
|
1 |
import os
|
2 |
+
import streamlit as st
|
3 |
+
import openai
|
4 |
+
from llama_index.core import SimpleDirectoryReader, VectorStoreIndex, StorageContext
|
5 |
+
from llama_index.core.node_parser import SentenceSplitter
|
6 |
+
from llama_index.retrievers.bm25 import BM25Retriever
|
7 |
+
from llama_index.core.retrievers import BaseRetriever
|
8 |
|
9 |
+
# Configuration
|
10 |
class Config:
|
11 |
OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
|
12 |
MODEL_NAME = "gpt-3.5-turbo"
|
13 |
EMBEDDING_MODEL = "text-embedding-3-small"
|
14 |
CHUNK_SIZE = 256
|
15 |
|
16 |
+
# Document Processing
|
|
|
|
|
|
|
|
|
17 |
class DocumentProcessor:
|
18 |
def __init__(self):
|
19 |
self.splitter = SentenceSplitter(chunk_size=Config.CHUNK_SIZE)
|
|
|
33 |
storage_context.docstore.add_documents(nodes)
|
34 |
return VectorStoreIndex(nodes=nodes, storage_context=storage_context), nodes
|
35 |
|
36 |
+
# Hybrid Retriever
|
|
|
|
|
|
|
37 |
class HybridRetriever(BaseRetriever):
|
38 |
def __init__(self, vector_retriever, bm25_retriever):
|
39 |
self.vector_retriever = vector_retriever
|
|
|
50 |
node_ids.add(n.node.node_id)
|
51 |
return all_nodes
|
52 |
|
53 |
+
# LLM Service
|
|
|
|
|
54 |
class LLMService:
|
55 |
def __init__(self, model_name):
|
56 |
self.model_name = model_name
|
|
|
84 |
)
|
85 |
return response.choices[0].message.content
|
86 |
|
87 |
+
# Main Application Class
|
|
|
|
|
|
|
|
|
|
|
|
|
88 |
class PromptOptimizationApp:
|
89 |
def __init__(self):
|
90 |
self.doc_processor = DocumentProcessor()
|