Update scrape.py

scrape.py

@@ -1,22 +1,22 @@
 import os
 
 from apify_client import ApifyClient
-from dotenv import load_dotenv
 from langchain.document_loaders import ApifyDatasetLoader
 from langchain.document_loaders.base import Document
 from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import Chroma
 
-# Load environment variables from a .env file
-load_dotenv()
+# Access variables and secrets as environment variables
+WEBSITE_URL = os.environ.get('WEBSITE_URL')
+OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')
+APIFY_API_TOKEN = os.environ.get('APIFY_API_TOKEN')
 
 if __name__ == '__main__':
-    apify_client = ApifyClient(os.environ.get('APIFY_API_TOKEN'))
-    website_url = os.environ.get('WEBSITE_URL')
-    print(f'Extracting data from "{website_url}". Please wait...')
+    apify_client = ApifyClient(APIFY_API_TOKEN)
+    print(f'Extracting data from "{WEBSITE_URL}". Please wait...')
     actor_run_info = apify_client.actor('apify/website-content-crawler').call(
-        run_input={'startUrls': [{'url': website_url}]}
+        run_input={'startUrls': [{'url': WEBSITE_URL}]}
     )
     print('Saving data into the vector database. Please wait...')
     loader = ApifyDatasetLoader(
@@ -29,7 +29,8 @@ if __name__ == '__main__':
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=100)
     docs = text_splitter.split_documents(documents)
 
-    embedding = OpenAIEmbeddings()
+    # Ensure the OPENAI_API_KEY is used correctly in OpenAIEmbeddings
+    embedding = OpenAIEmbeddings(api_key=OPENAI_API_KEY)
 
     vectordb = Chroma.from_documents(
         documents=docs,
@@ -37,4 +38,4 @@ if __name__ == '__main__':
         persist_directory='db2',
     )
     vectordb.persist()
-    print('All done!')
+    print('All done!')