Update scraper.py

scraper.py

@@ -6,9 +6,6 @@ from bs4 import BeautifulSoup
 import requests
 import time
 
-
-
-
 class Scraper:
     @staticmethod
     async def power_scrapper_2(url):
@@ -32,7 +29,10 @@ class Scraper:
             # Get page content (text from paragraphs and headers)
             page_content = await page.evaluate("""() => {
                 let elements = Array.from(document.querySelectorAll('body *'));
-                return elements.map(element => element.innerText).join('\\n');
+                return elements
+                    .filter(element => element.tagName.match(/^(P|H1|H2|H3|H4|H5|H6|LI|DIV|SPAN)$/i) && element.innerText.trim().length > 0)
+                    .map(element => element.innerText.trim())
+                    .join('\\n');
             }""")
 
             
@@ -99,7 +99,7 @@ class Scraper:
     async def scrape(url):
         try:
             headers = {'User-Agent': 'Mozilla/5.0'}
-            response = requests.get(url,timeout=5)
+            response = requests.get(url,timeout=3)
             soup = BeautifulSoup(response.content, 'html.parser')
     
             title = Scraper.get_title(soup)
@@ -109,8 +109,8 @@ class Scraper:
             if not links:
                 print("Running alternative scrapper")
                 links, text_content = await Scraper.power_scrapper_2(url)
+    
             return {"title": title, "URL": links, "Content": text_content}
         except:
-            print("Running alternative scrapper second time")
             title,links, text_content = await Scraper.power_scrapper_2(url)
-            return {"title": title, "URL": links, "Content": text_content}
+            return {"title": title, "URL": links, "Content": text_content}