SearchGPT: Production.

Signed-off-by: Hadad <[email protected]>

config.py

@@ -11,6 +11,12 @@ SEARXNG_ENDPOINT = "https://searx.stream/search"  # See the endpoint list at htt
 
 READER_ENDPOINT = "https://r.jina.ai/"
 
+USER_AGENT = (
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chromium/116.0.5845.97 Safari/537.36 "
+    "SearchGPT/1.0"
+)
+
 REQUEST_TIMEOUT = 300  # 5 minute
 
 MODEL = "gpt-4.1-nano"
@@ -186,6 +192,7 @@ ITERATION_METRICS = {
         0.06,
         0.07
     ],
+    "tools_reasoning_parsing": 1,
     "backoff_multiplier": 0.2
 }
 
@@ -244,197 +251,6 @@ ENABLE_TRUST_ENV = True  # AIOHTTP
 
 ENABLE_CONNECTOR_OWNER = True  # AIOHTTP
 
-OS = [
-    "Windows NT 10.0; Win64; x64",
-    "Macintosh; Intel Mac OS X 10_15_7",
-    "X11; Linux x86_64",
-    "Windows NT 11.0; Win64; x64",
-    "Macintosh; Intel Mac OS X 11_6_2"
-]
-
-OCTETS = [
-     1,   2,   3,   4,   5,   8,  12,  13,  14,  15,
-    16,  17,  18,  19,  20,  23,  24,  34,  35,  36,
-    37,  38,  39,  40,  41,  42,  43,  44,  45,  46,
-    47,  48,  49,  50,  51,  52,  53,  54,  55,  56,
-    57,  58,  59,  60,  61,  62,  63,  64,  65,  66,
-    67,  68,  69,  70,  71,  72,  73,  74,  75,  76,
-    77,  78,  79,  80,  81,  82,  83,  84,  85,  86,
-    87,  88,  89,  90,  91,  92,  93,  94,  95,  96,
-    97,  98,  99, 100, 101, 102, 103, 104, 105, 106,
-   107, 108, 109, 110, 111, 112, 113, 114, 115, 116,
-   117, 118, 119, 120, 121, 122, 123, 124, 125, 126,
-   128, 129, 130, 131, 132, 133, 134, 135, 136, 137,
-   138, 139, 140, 141, 142, 143, 144, 145, 146, 147,
-   148, 149, 150, 151, 152, 153, 154, 155, 156, 157,
-   158, 159, 160, 161, 162, 163, 164, 165, 166, 167,
-   168, 170, 171, 172, 173, 174, 175, 176, 177, 178,
-   179, 180, 181, 182, 183, 184, 185, 186, 187, 188,
-   189, 190, 191, 192, 193, 194, 195, 196, 197, 198,
-   199, 200, 201, 202, 203, 204, 205, 206, 207, 208,
-   209, 210, 211, 212, 213, 214, 215, 216, 217, 218,
-   219, 220, 221, 222, 223
-]
-
-BROWSERS = [
-    "Chrome",
-    "Firefox",
-    "Safari",
-    "Edge",
-    "Opera"
-]
-
-CHROME_VERSIONS = [
-    "120.0.0.0",
-    "119.0.0.0",
-    "118.0.0.0",
-    "117.0.0.0",
-    "116.0.0.0"
-]
-
-FIREFOX_VERSIONS = [
-    "121.0",
-    "120.0",
-    "119.0",
-    "118.0",
-    "117.0"
-]
-
-SAFARI_VERSIONS = [
-    "17.1",
-    "17.0",
-    "16.6",
-    "16.5",
-    "16.4",
-]
-
-EDGE_VERSIONS = [
-    "120.0.2210.91",
-    "119.0.2151.97",
-    "118.0.2088.76",
-    "117.0.2045.60",
-    "116.0.1938.81"
-]
-
-DOMAINS = [
-    "google.com",
-    "bing.com",
-    "yahoo.com",
-    "duckduckgo.com",
-    "baidu.com",
-    "yandex.com",
-    "facebook.com",
-    "twitter.com",
-    "linkedin.com",
-    "reddit.com",
-    "youtube.com",
-    "wikipedia.org",
-    "amazon.com",
-    "github.com",
-    "stackoverflow.com",
-    "medium.com",
-    "quora.com",
-    "pinterest.com",
-    "instagram.com",
-    "tumblr.com"
-]
-
-PROTOCOLS = [
-    "https://",
-    "https://www."
-]
-
-SEARCH_ENGINES = [
-    "https://www.google.com/search?q=",
-    "https://www.bing.com/search?q=",
-    "https://search.yahoo.com/search?p=",
-    "https://duckduckgo.com/?q=",
-    "https://www.baidu.com/s?wd=",
-    "https://yandex.com/search/?text=",
-    "https://www.google.co.uk/search?q=",
-    "https://www.google.ca/search?q=",
-    "https://www.google.com.au/search?q=",
-    "https://www.google.de/search?q=",
-    "https://www.google.fr/search?q=",
-    "https://www.google.co.jp/search?q=",
-    "https://www.google.com.br/search?q=",
-    "https://www.google.co.in/search?q=",
-    "https://www.google.ru/search?q=",
-    "https://www.google.it/search?q="
-]
-
-KEYWORDS = [
-    "news",
-    "weather",
-    "sports",
-    "technology",
-    "science",
-    "health",
-    "finance",
-    "entertainment",
-    "travel",
-    "food",
-    "education",
-    "business",
-    "politics",
-    "culture",
-    "history",
-    "music",
-    "movies",
-    "games",
-    "books",
-    "art"
-]
-
-COUNTRIES = [
-    "US", "GB", "CA", "AU", "DE", "FR", "JP", "BR", "IN", "RU",
-    "IT", "ES", "MX", "NL", "SE", "NO", "DK", "FI", "PL", "TR",
-    "KR", "SG", "HK", "TW", "TH", "ID", "MY", "PH", "VN", "AR",
-    "CL", "CO", "PE", "VE", "EG", "ZA", "NG", "KE", "MA", "DZ",
-    "TN", "IL", "AE", "SA", "QA", "KW", "BH", "OM", "JO", "LB"
-]
-
-LANGUAGES = [
-    "en-US", "en-GB", "en-CA", "en-AU", "de-DE", "fr-FR", "ja-JP",
-    "pt-BR", "hi-IN", "ru-RU", "it-IT", "es-ES", "es-MX", "nl-NL",
-    "sv-SE", "no-NO", "da-DK", "fi-FI", "pl-PL", "tr-TR", "ko-KR",
-    "zh-CN", "zh-TW", "th-TH", "id-ID", "ms-MY", "fil-PH", "vi-VN",
-    "es-AR", "es-CL", "es-CO", "es-PE", "es-VE", "ar-EG", "en-ZA",
-    "en-NG", "sw-KE", "ar-MA", "ar-DZ", "ar-TN", "he-IL", "ar-AE",
-    "ar-SA", "ar-QA", "ar-KW", "ar-BH", "ar-OM", "ar-JO", "ar-LB"
-]
-
-TIMEZONES = [
-    "America/New_York",
-    "America/Chicago",
-    "America/Los_Angeles",
-    "America/Denver",
-    "Europe/London",
-    "Europe/Paris",
-    "Europe/Berlin",
-    "Europe/Moscow",
-    "Asia/Tokyo",
-    "Asia/Shanghai",
-    "Asia/Hong_Kong",
-    "Asia/Singapore",
-    "Asia/Seoul",
-    "Asia/Mumbai",
-    "Asia/Dubai",
-    "Australia/Sydney",
-    "Australia/Melbourne",
-    "America/Toronto",
-    "America/Vancouver",
-    "America/Mexico_City",
-    "America/Sao_Paulo",
-    "America/Buenos_Aires",
-    "Africa/Cairo",
-    "Africa/Johannesburg",
-    "Africa/Lagos",
-    "Africa/Nairobi",
-    "Pacific/Auckland",
-    "Pacific/Honolulu"
-]
-
 DESCRIPTION = (
     "<b>SearchGPT</b> is <b>ChatGPT</b> with real-time web search capabilities "
     "and the ability to read content directly from a URL.<br><br>"

src/core/__init__.py

@@ -3,10 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-from .web_loader import WebLoader
 from .web_configuration import WebConfiguration
 
 __all__ = [
-    'WebLoader',
     'WebConfiguration'
 ]

src/core/web_loader.py

@@ -1,160 +0,0 @@
-#
-# SPDX-FileCopyrightText: Hadad <[email protected]>
-# SPDX-License-Identifier: Apache-2.0
-#
-
-import random
-import threading
-from collections import deque
-from config import (
-    OS,
-    OCTETS,
-    BROWSERS,
-    CHROME_VERSIONS,
-    FIREFOX_VERSIONS,
-    SAFARI_VERSIONS,
-    EDGE_VERSIONS,
-    DOMAINS,
-    PROTOCOLS,
-    SEARCH_ENGINES,
-    KEYWORDS,
-    COUNTRIES,
-    LANGUAGES,
-    TIMEZONES
-)
-
-class WebLoader:
-    def __init__(self):
-        self.ipv4_pool = deque(maxlen=1000)
-        self.ipv6_pool = deque(maxlen=1000)
-        self.user_agent_pool = deque(maxlen=500)
-        self.origin_pool = deque(maxlen=500)
-        self.referrer_pool = deque(maxlen=500)
-        self.location_pool = deque(maxlen=500)
-        self.lock = threading.Lock()
-        self.running = True
-        
-    def generate_ipv4(self):
-        while len(self.ipv4_pool) < 1000 and self.running:
-            with self.lock:
-                self.ipv4_pool.append(
-                    f"{random.choice(OCTETS)}.{random.randint(0, 255)}."
-                    f"{random.randint(0, 255)}.{random.randint(1, 254)}"
-                )
-            
-    def generate_ipv6(self):
-        while len(self.ipv6_pool) < 1000 and self.running:
-            with self.lock:
-                self.ipv6_pool.append(
-                    ":".join([f"{random.randint(0, 65535):04x}" for _ in range(8)])
-                )
-            
-    def generate_user_agents(self):
-        while len(self.user_agent_pool) < 500 and self.running:
-            with self.lock:
-                match random.choice(BROWSERS):
-                    case "Chrome":
-                        self.user_agent_pool.append(
-                            f"Mozilla/5.0 ({random.choice(OS)}) AppleWebKit/537.36 "
-                            f"(KHTML, like Gecko) Chrome/{random.choice(CHROME_VERSIONS)} Safari/537.36"
-                        )
-                    case "Firefox":
-                        self.user_agent_pool.append(
-                            f"Mozilla/5.0 ({random.choice(OS)}) Gecko/20100101 "
-                            f"Firefox/{random.choice(FIREFOX_VERSIONS)}"
-                        )
-                    case "Safari":
-                        self.user_agent_pool.append(
-                            f"Mozilla/5.0 ({random.choice(OS)}) AppleWebKit/"
-                            f"{600 + random.randint(0, 15)}.{random.randint(1, 9)}.{random.randint(1, 20)} "
-                            f"(KHTML, like Gecko) Version/{random.choice(SAFARI_VERSIONS)} "
-                            f"Safari/{600 + random.randint(0, 15)}.{random.randint(1, 9)}.{random.randint(1, 20)}"
-                        )
-                    case "Edge":
-                        self.user_agent_pool.append(
-                            f"Mozilla/5.0 ({random.choice(OS)}) AppleWebKit/537.36 "
-                            f"(KHTML, like Gecko) Chrome/{random.choice(EDGE_VERSIONS).split('.')[0]}.0.0.0 "
-                            f"Safari/537.36 Edg/{random.choice(EDGE_VERSIONS)}"
-                        )
-                    case _:
-                        self.user_agent_pool.append(
-                            f"Mozilla/5.0 ({random.choice(OS)}) AppleWebKit/537.36 "
-                            f"(KHTML, like Gecko) Chrome/{random.randint(70, 100)}.0."
-                            f"{random.randint(3000, 5000)}.{random.randint(50, 150)} "
-                            f"Safari/537.36 OPR/{random.randint(80, 106)}.0.0.0"
-                        )
-            
-    def generate_origins(self):
-        while len(self.origin_pool) < 500 and self.running:
-            with self.lock:
-                self.origin_pool.append(
-                    f"{random.choice(PROTOCOLS)}{random.choice(DOMAINS)}"
-                )
-            
-    def generate_referrers(self):
-        while len(self.referrer_pool) < 500 and self.running:
-            with self.lock:
-                self.referrer_pool.append(
-                    f"{random.choice(SEARCH_ENGINES)}{random.choice(KEYWORDS)}"
-                )
-            
-    def generate_locations(self):
-        while len(self.location_pool) < 500 and self.running:
-            with self.lock:
-                self.location_pool.append({
-                    "country": random.choice(COUNTRIES),
-                    "language": random.choice(LANGUAGES),
-                    "timezone": random.choice(TIMEZONES)
-                })
-            
-    def get_ipv4(self):
-        with self.lock:
-            if self.ipv4_pool:
-                return self.ipv4_pool[random.randint(0, len(self.ipv4_pool) - 1)]
-        return (
-            f"{random.randint(1, 223)}.{random.randint(0, 255)}."
-            f"{random.randint(0, 255)}.{random.randint(1, 254)}"
-        )
-        
-    def get_ipv6(self):
-        with self.lock:
-            if self.ipv6_pool:
-                return self.ipv6_pool[random.randint(0, len(self.ipv6_pool) - 1)]
-        return ":".join([f"{random.randint(0, 65535):04x}" for _ in range(8)])
-        
-    def get_user_agent(self):
-        with self.lock:
-            if self.user_agent_pool:
-                return self.user_agent_pool[random.randint(0, len(self.user_agent_pool) - 1)]
-        
-    def get_origin(self):
-        with self.lock:
-            if self.origin_pool:
-                return self.origin_pool[random.randint(0, len(self.origin_pool) - 1)]
-        
-    def get_referrer(self):
-        with self.lock:
-            if self.referrer_pool:
-                return self.referrer_pool[random.randint(0, len(self.referrer_pool) - 1)]
-        
-    def get_location(self):
-        with self.lock:
-            if self.location_pool:
-                return self.location_pool[random.randint(0, len(self.location_pool) - 1)]
-        
-    def start_engine(self):
-        for target in [
-            self.generate_ipv4,
-            self.generate_ipv6,
-            self.generate_user_agents,
-            self.generate_origins,
-            self.generate_referrers,
-            self.generate_locations
-        ]:
-            threading.Thread(target=target, daemon=True).start()
-            
-    def stop(self):
-        self.running = False
-
-web_loader = WebLoader()
-web_loader.start_engine()

src/engine/browser_engine.py

@@ -15,9 +15,9 @@ from config import (
     TCP_CONNECTOR_FORCE_CLOSE,
     TCP_CONNECTOR_ENABLE_CLEANUP,
     ENABLE_TRUST_ENV,
-    ENABLE_CONNECTOR_OWNER
+    ENABLE_CONNECTOR_OWNER,
+    USER_AGENT
 )
-from ..core.web_loader import web_loader
 
 class BrowserEngine:
     def __init__(self, configuration):
@@ -25,29 +25,7 @@ class BrowserEngine:
 
     def generate_headers(self):
         return {
-            "User-Agent": web_loader.get_user_agent(),
-            "X-Forwarded-For": f"{web_loader.get_ipv4()}, {web_loader.get_ipv6()}",
-            "X-Real-IP": web_loader.get_ipv4(),
-            "X-Originating-IP": web_loader.get_ipv4(),
-            "X-Remote-IP": web_loader.get_ipv4(),
-            "X-Remote-Addr": web_loader.get_ipv4(),
-            "X-Client-IP": web_loader.get_ipv4(),
-            "X-Forwarded-Host": web_loader.get_origin().replace("https://", "").replace("http://", ""),
-            "Origin": web_loader.get_origin(),
-            "Referer": web_loader.get_referrer(),
-            "Accept-Language": f"{web_loader.get_location()['language']},en;q=0.9",
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-            "Accept-Encoding": "gzip, deflate, br",
-            "DNT": "1",
-            "Connection": "keep-alive",
-            "Upgrade-Insecure-Requests": "1",
-            "Sec-Fetch-Dest": "document",
-            "Sec-Fetch-Mode": "navigate",
-            "Sec-Fetch-Site": "cross-site",
-            "Sec-Fetch-User": "?1",
-            "Cache-Control": "max-age=0",
-            "X-Country": web_loader.get_location()['country'],
-            "X-Timezone": web_loader.get_location()['timezone']
+            "User-Agent": USER_AGENT
         }
 
     def web_selector(self, search_query: str, search_provider: str = "google"):

src/processor/message_processor.py

@@ -29,7 +29,6 @@ def searchgpt_playground(user_message, chat_history):
 
         conversation_messages = setup_response(chat_history, user_message)
         tool_response = ""
-        tools_done = False
         
         for tool_update in tools_setup(
             server=server,
@@ -44,7 +43,6 @@ def searchgpt_playground(user_message, chat_history):
             else:
                 conversation_messages = tool_update[0]
                 tool_response = tool_update[1]
-                tools_done = tool_update[2]
 
         if tool_response:
             yield tool_response + "\n\n"
@@ -52,9 +50,7 @@ def searchgpt_playground(user_message, chat_history):
         final_response_generator = generate_response(
             server=server,
             model_name=MODEL,
-            conversation_messages=conversation_messages,
-            tool_definitions=local_tools(),
-            tools_done=tools_done
+            conversation_messages=conversation_messages
         )
         
         for final_response in final_response_generator:

src/processor/response/generator.py

@@ -12,9 +12,7 @@ from config import (
 def generate_response(
     server,
     model_name,
-    conversation_messages,
-    tool_definitions,
-    tools_done=False
+    conversation_messages
 ):
     response_generator = ""
     
@@ -22,8 +20,6 @@ def generate_response(
         response = server.chat.completions.create(
             model=model_name,
             messages=conversation_messages,
-            tools=tool_definitions if not tools_done else None,
-            tool_choice="none",
             temperature=CHAT_TEMPERATURE,
             stream=STREAM
         )

src/processor/tools/interaction.py

@@ -62,6 +62,7 @@ def tools_setup(server, model_name, conversation_messages, tool_definitions, sea
                         for i in range(0, len(tool_reasoning(tool_invocation.function.name, tools_parser(tool_invocation.function.arguments)[0], "parsing")) + 1):
                             ITERATION_METRICS["logs_generator"] = styles(reasoning_interfaces(tool_reasoning(tool_invocation.function.name, tools_parser(tool_invocation.function.arguments)[0], "parsing"), i), expanded=True)
                             yield ITERATION_METRICS["logs_generator"]
+                        time.sleep(ITERATION_METRICS["tools_reasoning_parsing"])
                         
                         for i in range(0, len(tool_reasoning(tool_invocation.function.name, tools_parser(tool_invocation.function.arguments)[0], "executing")) + 1):
                             ITERATION_METRICS["logs_generator"] = styles(reasoning_interfaces(tool_reasoning(tool_invocation.function.name, tools_parser(tool_invocation.function.arguments)[0], "executing"), i), expanded=True)