feat: simple scrape with css_selector

app.py

@@ -5,7 +5,7 @@ import base64
 from io import BytesIO
 from PIL import Image
 from crewai import Agent, Task, Crew, Process, LLM
-from crewai_tools import ScrapeWebsiteTool
+from crewai_tools import ScrapeElementFromWebsiteTool
 from dotenv import load_dotenv
 from stealth_scrape_tool import StealthScrapeTool
 from image_generator_tool import GenerateImageTool
@@ -20,7 +20,7 @@ class SocialMediaCrew:
         self.natura_api_token = natura_api_token
         self.openai_base_url = openai_base_url
         self.openai_model_name = openai_model_name
-        self.scrape_tool = ScrapeWebsiteTool()
+        self.scrape_tool = ScrapeElementFromWebsiteTool()
         self.calculate_discounted_price_tool = CalculateDiscountedPriceTool()
         self.calculate_discount_value_tool = CalculateDiscountValueTool()
         self.image_generator_tool = GenerateImageTool()
@@ -89,7 +89,7 @@ class SocialMediaCrew:
         return merchant, css_selector, short_url
 
     def _create_analyze_product_task(self, product_url: str, css_selector: str, main_cupom_discount_percentage: float, short_url: str, original_price: float, discounted_price: float) -> Task:
-        task_description = (f"1. Scrape the content of the URL: {product_url} using the 'scrape_tool'.\n"
+        task_description = (f"1. Scrape the content of the URL: {product_url} using the 'scrape_tool' with param `css_element` as `{css_selector}`.\n"
                             "2. Extract the product name, key characteristics, and any other relevant DISCOUNT available.\n")
 
         if original_price is not None and original_price > 0 and discounted_price is not None and discounted_price > 0:
@@ -267,7 +267,7 @@ with gr.Blocks() as demo:
         openai_key_input = gr.Textbox(label="OPENAI_API_KEY", type="password", value=os.getenv("OPENAI_API_KEY", ""))
         natura_token_input = gr.Textbox(label="NATURA_API_TOKEN", type="password", value=os.getenv("NATURA_API_TOKEN", ""))
         openai_base_url_input = gr.Textbox(label="OPENAI_BASE_URL", value=os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1"))
-        openai_model_name_input = gr.Textbox(label="OPENAI_MODEL_NAME", value=os.getenv("OPENAI_MODEL_NAME", "gpt-4.1"))
+        openai_model_name_input = gr.Textbox(label="OPENAI_MODEL_NAME", value=os.getenv("OPENAI_MODEL_NAME", "gpt-4.1-mini"))
         
         clean_env_vars()
         # No save button needed as keys are passed directly

stealth_scrape_tool.py

@@ -17,7 +17,7 @@ class StealthScrapeTool(BaseTool):
                 print(f"StealthScrapeTool: Starting scraping for {website_url}...")
                 print(f"StealthScrapeTool: Navigating to {website_url}")
                 await page.goto(website_url, timeout=120000)
-                await asyncio.sleep(5)
+                await asyncio.sleep(1)
 
                 # Scroll to the bottom of the page repeatedly to load all dynamic content
                 print("StealthScrapeTool: Scrolling through the page to load dynamic content...")
@@ -33,7 +33,7 @@ class StealthScrapeTool(BaseTool):
                     await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
                     print("StealthScrapeTool: Scrolled. Waiting for content to load...")
 
-                    await asyncio.sleep(5)
+                    await asyncio.sleep(1)
 
                     print("StealthScrapeTool: Getting new scrollHeight...")
                     new_height = await page.evaluate("document.body.scrollHeight")