Spaces:
Running
Running
File size: 1,788 Bytes
372531f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 |
import asyncio
from dotenv import load_dotenv
from gpt_researcher.config.config import Config
from gpt_researcher.actions.retriever import get_retrievers
from gpt_researcher.skills.researcher import ResearchConductor
import pprint
# Load environment variables from .env file
load_dotenv()
async def test_scrape_data_by_query():
# Initialize the Config object
config = Config()
# Retrieve the retrievers based on the current configuration
retrievers = get_retrievers({}, config)
print("Retrievers:", retrievers)
# Create a mock researcher object with necessary attributes
class MockResearcher:
def init(self):
self.retrievers = retrievers
self.cfg = config
self.verbose = True
self.websocket = None
self.scraper_manager = None # Mock or implement scraper manager
self.vector_store = None # Mock or implement vector store
researcher = MockResearcher()
research_conductor = ResearchConductor(researcher)
# print('research_conductor',dir(research_conductor))
# print('MockResearcher',dir(researcher))
# Define a sub-query to test
sub_query = "design patterns for autonomous ai agents"
# Iterate through all retrievers
for retriever_class in retrievers:
# Instantiate the retriever with the sub-query
retriever = retriever_class(sub_query)
# Perform the search using the current retriever
search_results = await asyncio.to_thread(
retriever.search, max_results=10
)
print("\033[35mSearch results:\033[0m")
pprint.pprint(search_results, indent=4, width=80)
if __name__ == "__main__":
asyncio.run(test_scrape_data_by_query()) |