Better Scraping

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,2 +1 @@
-__pychache__
 .env

README.md

@@ -128,4 +128,5 @@ Special thanks to **Hugging Face** for hosting this space and **Groq AI** for th
 ### **🚀 Ready to get company insights?**
 Run the scraper and generate **detailed company reports effortlessly**! 🔍
 
+
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -32,11 +32,12 @@ scraper = cloudscraper.create_scraper()
 # Headers to mimic real browser requests
 HEADERS = {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-    "Accept-Language": "en-US,en;q=0.9",
-    "Referer": "https://www.google.com/"
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
 }
 
-#  **Extract Links**
+# ✅ **Extract Links**
 def get_valid_links(base_url):
     """Extracts all internal links, including footer and JavaScript-rendered links."""
     try:
@@ -58,7 +59,7 @@ def get_valid_links(base_url):
 
         return links
     except requests.exceptions.RequestException as e:
-        print(f" Error fetching links: {e}")
+        print(f"❌ Error fetching links: {e}")
         return set()
 
 def check_footer_links(soup):
@@ -91,10 +92,10 @@ def get_links_with_selenium(url):
 
         return links
     except Exception as e:
-        print(f" Selenium Error: {e}")
+        print(f"❌ Selenium Error: {e}")
         return set()
 
-#  **Scrape Pages**
+# ✅ **Scrape Pages**
 def scrape_page(url):
     """Scrapes a webpage, using Selenium if necessary."""
     try:
@@ -125,21 +126,21 @@ def scrape_with_selenium(url):
 
         return extract_text(soup)
     except Exception as e:
-        return f" Selenium Scraping Error: {e}"
+        return f"❌ Selenium Scraping Error: {e}"
     
 def extract_text(soup):
     """Extracts **all** meaningful text from HTML content, including dynamic elements."""
     
-    #  Extracts all text from the HTML, not just specific tags
+    # ✅ Extracts all text from the HTML, not just specific tags
     all_text = soup.get_text(separator="\n", strip=True)
     
-    #  Removes duplicate lines & unwanted spaces
+    # ✅ Removes duplicate lines & unwanted spaces
     unique_lines = set(all_text.split("\n"))
     cleaned_text = "\n".join(line for line in unique_lines if len(line) > 3)  # Exclude tiny fragments
     
     return cleaned_text
 
-#  **Chunking for Large AI Requests**
+# ✅ **Chunking for Large AI Requests**
 def split_into_chunks(text, chunk_size):
     """Splits long content into manageable chunks for AI processing."""
     words = text.split()
@@ -160,7 +161,7 @@ def split_into_chunks(text, chunk_size):
 
     return chunks
 
-#  **AI-Powered Company Breakdown**
+# ✅ **AI-Powered Company Breakdown**
 def generate_detailed_company_info(company_data):
     """Generates an in-depth company breakdown with AI."""
     
@@ -220,7 +221,7 @@ def generate_detailed_company_info(company_data):
         response = chain.invoke({"text": user_prompt_template})
         return response.content
 
-#  **Streamlit UI**
+# ✅ **Streamlit UI**
 def main():
     st.title("🚀 AI-Powered Company Website Scraper")
     base_url = st.text_input("🔗 Enter Website URL", "")

requirements.txt

@@ -1,6 +1,6 @@
 urllib3==2.3.0
-requests==2.32.3
 selenium==4.21.0
+requests==2.32.3
 streamlit==1.41.1
 cloudscraper==1.2.71
 python-dotenv==1.0.1