initial commit

requirements.txt

@@ -0,0 +1,74 @@
+aiofiles==23.2.1
+annotated-types==0.7.0
+anyio==4.4.0
+beautifulsoup4==4.12.3
+bs4==0.0.2
+cachetools==5.5.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+exceptiongroup==1.2.1
+fastapi==0.115.3
+ffmpy==0.4.0
+filelock==3.15.4
+firecrawl-py==0.0.12
+fpdf==1.7.2
+fsspec==2024.6.1
+gradio==5.3.0
+gradio_client==1.4.2
+grpcio==1.66.0
+grpcio-status==1.62.3
+h11==0.14.0
+httpcore==1.0.5
+httplib2==0.22.0
+httpx==0.26.0
+huggingface-hub==0.26.1
+idna==3.7
+iniconfig==2.0.0
+Jinja2==3.1.4
+Markdown==3.7
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+mdurl==0.1.2
+numpy==2.1.2
+orjson==3.10.10
+packaging==24.0
+pandas==2.2.3
+pillow==10.4.0
+pluggy==1.5.0
+proto-plus==1.24.0
+protobuf==4.25.4
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pydantic==2.7.2
+pydantic_core==2.18.3
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.1.2
+pytest==8.2.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.12
+pytz==2024.2
+PyYAML==6.0.1
+requests==2.32.3
+responses==0.24.1
+rich==13.9.3
+rsa==4.9
+ruff==0.7.1
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+soupsieve==2.5
+starlette==0.41.0
+tomli==2.0.1
+tomlkit==0.12.0
+tqdm==4.66.5
+typer==0.12.5
+typing_extensions==4.12.0
+tzdata==2024.2
+uritemplate==4.1.1
+urllib3==2.2.1
+uvicorn==0.32.0
+websockets==12.0

uqnaapp.py

@@ -0,0 +1,89 @@
+import gradio as gr
+import os
+import time
+import json
+import requests
+from firecrawl import FirecrawlApp
+
+# Configuration
+API_KEY = "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"
+FIRECRAWL_API_KEY = "fc-b9c9f8db590f4ea99b122d93eaf5160b"
+API_URL = "https://openrouter.ai/api/v1/chat/completions"
+SCRAPED_DATA_DIR = "scraped_data"
+HARDCODED_DATA_FILE = "innotechtitans_data.json"
+os.makedirs(SCRAPED_DATA_DIR, exist_ok=True)
+
+# Function to scrape website data
+def scrape_data(url):
+    app = FirecrawlApp(api_key=FIRECRAWL_API_KEY)
+    scraped_data = app.scrape_url(url, {'pageOptions': {'onlyMainContent': True}})
+    
+    if 'markdown' not in scraped_data:
+        return "Error: Unable to scrape data."
+    
+    domain_name = url.split("//")[-1].split("/")[0]
+    file_path = os.path.join(SCRAPED_DATA_DIR, f"{domain_name}.md")
+    
+    with open(file_path, 'w', encoding='utf-8') as file:
+        file.write(scraped_data['markdown'])
+    
+    return f"✅ Scraped data saved as {domain_name}.md. Load it to proceed."
+
+# Function to load data from a markdown file
+def load_data(file_name):
+    file_path = os.path.join(SCRAPED_DATA_DIR, file_name)
+    if not os.path.exists(file_path):
+        return "❌ Error: File not found."
+    
+    with open(file_path, 'r', encoding='utf-8') as file:
+        global loaded_data
+        loaded_data = file.read()
+    
+    return "✅ Data loaded successfully. You can now ask questions."
+
+# Function to send a query to LLM
+def ask_question(question):
+    if not loaded_data:
+        return "⚠️ No data loaded. Please scrape a website or load data first."
+    
+    headers = {"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"}
+    payload = {
+        "model": "deepseek/deepseek-chat:free",
+        "messages": [{"role": "user", "content": f"{loaded_data}\n\n{question}"}]
+    }
+    
+    response = requests.post(API_URL, json=payload, headers=headers)
+    
+    if response.status_code == 200:
+        return response.json().get("choices", [{}])[0].get("message", {}).get("content", "No response.")
+    return "❌ Error: Unable to generate response."
+
+# Gradio Interface
+with gr.Blocks(theme=gr.themes.Default()) as demo:
+    gr.Markdown("""
+        <h1 style='text-align: center;'>🕸️ Web Scraper & AI QnA</h1>
+        <p style='text-align: center; font-size: 18px;'>Scrape websites and ask AI-powered questions!</p>
+    """)
+    
+    with gr.Tab("Scrape Website"):
+        with gr.Row():
+            url_input = gr.Textbox(label="🌐 Website URL", placeholder="Enter URL to scrape")
+            scrape_button = gr.Button("🚀 Scrape", variant="primary")
+        scrape_output = gr.Markdown()
+        scrape_button.click(scrape_data, inputs=[url_input], outputs=[scrape_output])
+    
+    with gr.Tab("Load Data"):
+        with gr.Row():
+            file_input = gr.Textbox(label="📂 Markdown File Name", placeholder="Enter filename (e.g., site.md)")
+            load_button = gr.Button("📥 Load", variant="primary")
+        load_output = gr.Markdown()
+        load_button.click(load_data, inputs=[file_input], outputs=[load_output])
+    
+    with gr.Tab("Ask AI"):
+        with gr.Row():
+            question_input = gr.Textbox(label="❓ Ask a Question", placeholder="Ask based on loaded data")
+            ask_button = gr.Button("💬 Ask", variant="primary")
+        answer_output = gr.Markdown()
+        ask_button.click(ask_question, inputs=[question_input], outputs=[answer_output])
+
+demo.launch()