Merge pull request #2599 from open-webui/websearch

feat: websearch

backend/apps/rag/main.py

@@ -11,7 +11,7 @@ from fastapi.middleware.cors import CORSMiddleware
 import os, shutil, logging, re
 
 from pathlib import Path
-from typing import List
+from typing import List, Union, Sequence
 
 from chromadb.utils.batch_utils import create_batches
 
@@ -59,6 +59,7 @@ from apps.rag.utils import (
     query_doc_with_hybrid_search,
     query_collection,
     query_collection_with_hybrid_search,
+    search_web,
 )
 
 from utils.misc import (
@@ -95,6 +96,7 @@ from config import (
     RAG_TEMPLATE,
     ENABLE_RAG_LOCAL_WEB_FETCH,
     YOUTUBE_LOADER_LANGUAGE,
+    RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
     AppConfig,
 )
 
@@ -201,6 +203,10 @@ class UrlForm(CollectionNameForm):
     url: str
 
 
+class SearchForm(CollectionNameForm):
+    query: str
+
+
 @app.get("/")
 async def get_status():
     return {
@@ -589,8 +595,20 @@ def store_web(form_data: UrlForm, user=Depends(get_current_user)):
         )
 
 
-def get_web_loader(url: str, verify_ssl: bool = True):
+def get_web_loader(url: Union[str, Sequence[str]], verify_ssl: bool = True):
     # Check if the URL is valid
+    if not validate_url(url):
+        raise ValueError(ERROR_MESSAGES.INVALID_URL)
+    return WebBaseLoader(
+        url,
+        verify_ssl=verify_ssl,
+        requests_per_second=RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
+        continue_on_failure=True,
+    )
+
+
+def validate_url(url: Union[str, Sequence[str]]):
+    if isinstance(url, str):
         if isinstance(validators.url(url), validators.ValidationError):
             raise ValueError(ERROR_MESSAGES.INVALID_URL)
         if not ENABLE_RAG_LOCAL_WEB_FETCH:
@@ -606,7 +624,11 @@ def get_web_loader(url: str, verify_ssl: bool = True):
             for ip in ipv6_addresses:
                 if validators.ipv6(ip, private=True):
                     raise ValueError(ERROR_MESSAGES.INVALID_URL)
-    return WebBaseLoader(url, verify_ssl=verify_ssl)
+        return True
+    elif isinstance(url, Sequence):
+        return all(validate_url(u) for u in url)
+    else:
+        return False
 
 
 def resolve_hostname(hostname):
@@ -620,6 +642,39 @@ def resolve_hostname(hostname):
     return ipv4_addresses, ipv6_addresses
 
 
+@app.post("/web/search")
+def store_web_search(form_data: SearchForm, user=Depends(get_current_user)):
+    try:
+        try:
+            web_results = search_web(form_data.query)
+        except Exception as e:
+            log.exception(e)
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.WEB_SEARCH_ERROR,
+            )
+        urls = [result.link for result in web_results]
+        loader = get_web_loader(urls)
+        data = loader.load()
+
+        collection_name = form_data.collection_name
+        if collection_name == "":
+            collection_name = calculate_sha256_string(form_data.query)[:63]
+
+        store_data_in_vector_db(data, collection_name, overwrite=True)
+        return {
+            "status": True,
+            "collection_name": collection_name,
+            "filenames": urls,
+        }
+    except Exception as e:
+        log.exception(e)
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.DEFAULT(e),
+        )
+
+
 def store_data_in_vector_db(data, collection_name, overwrite: bool = False) -> bool:
 
     text_splitter = RecursiveCharacterTextSplitter(

backend/apps/rag/search/brave.py

+import logging
+
+import requests
+
+from apps.rag.search.main import SearchResult
+from config import SRC_LOG_LEVELS, RAG_WEB_SEARCH_RESULT_COUNT
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_brave(api_key: str, query: str) -> list[SearchResult]:
+    """Search using Brave's Search API and return the results as a list of SearchResult objects.
+
+    Args:
+        api_key (str): A Brave Search API key
+        query (str): The query to search for
+    """
+    url = "https://api.search.brave.com/res/v1/web/search"
+    headers = {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip",
+        "X-Subscription-Token": api_key,
+    }
+    params = {"q": query, "count": RAG_WEB_SEARCH_RESULT_COUNT}
+
+    response = requests.get(url, headers=headers, params=params)
+    response.raise_for_status()
+
+    json_response = response.json()
+    results = json_response.get("web", {}).get("results", [])
+    return [
+        SearchResult(
+            link=result["url"], title=result.get("title"), snippet=result.get("snippet")
+        )
+        for result in results[:RAG_WEB_SEARCH_RESULT_COUNT]
+    ]

backend/apps/rag/search/google_pse.py

+import json
+import logging
+
+import requests
+
+from apps.rag.search.main import SearchResult
+from config import SRC_LOG_LEVELS, RAG_WEB_SEARCH_RESULT_COUNT
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_google_pse(
+    api_key: str, search_engine_id: str, query: str
+) -> list[SearchResult]:
+    """Search using Google's Programmable Search Engine API and return the results as a list of SearchResult objects.
+
+    Args:
+        api_key (str): A Programmable Search Engine API key
+        search_engine_id (str): A Programmable Search Engine ID
+        query (str): The query to search for
+    """
+    url = "https://www.googleapis.com/customsearch/v1"
+
+    headers = {"Content-Type": "application/json"}
+    params = {
+        "cx": search_engine_id,
+        "q": query,
+        "key": api_key,
+        "num": RAG_WEB_SEARCH_RESULT_COUNT,
+    }
+
+    response = requests.request("GET", url, headers=headers, params=params)
+    response.raise_for_status()
+
+    json_response = response.json()
+    results = json_response.get("items", [])
+    return [
+        SearchResult(
+            link=result["link"],
+            title=result.get("title"),
+            snippet=result.get("snippet"),
+        )
+        for result in results
+    ]

backend/apps/rag/search/main.py

+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class SearchResult(BaseModel):
+    link: str
+    title: Optional[str]
+    snippet: Optional[str]

backend/apps/rag/search/searxng.py

+import logging
+
+import requests
+
+from apps.rag.search.main import SearchResult
+from config import SRC_LOG_LEVELS, RAG_WEB_SEARCH_RESULT_COUNT
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_searxng(query_url: str, query: str) -> list[SearchResult]:
+    """Search a SearXNG instance for a query and return the results as a list of SearchResult objects.
+
+    Args:
+        query_url (str): The URL of the SearXNG instance to search. Must contain "<query>" as a placeholder
+        query (str): The query to search for
+    """
+    url = query_url.replace("<query>", query)
+    if "&format=json" not in url:
+        url += "&format=json"
+    log.debug(f"searching {url}")
+
+    r = requests.get(
+        url,
+        headers={
+            "User-Agent": "Open WebUI (https://github.com/open-webui/open-webui) RAG Bot",
+            "Accept": "text/html",
+            "Accept-Encoding": "gzip, deflate",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Connection": "keep-alive",
+        },
+    )
+    r.raise_for_status()
+
+    json_response = r.json()
+    results = json_response.get("results", [])
+    sorted_results = sorted(results, key=lambda x: x.get("score", 0), reverse=True)
+    return [
+        SearchResult(
+            link=result["url"], title=result.get("title"), snippet=result.get("content")
+        )
+        for result in sorted_results[:RAG_WEB_SEARCH_RESULT_COUNT]
+    ]

backend/apps/rag/search/serper.py

+import json
+import logging
+
+import requests
+
+from apps.rag.search.main import SearchResult
+from config import SRC_LOG_LEVELS, RAG_WEB_SEARCH_RESULT_COUNT
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_serper(api_key: str, query: str) -> list[SearchResult]:
+    """Search using serper.dev's API and return the results as a list of SearchResult objects.
+
+    Args:
+        api_key (str): A serper.dev API key
+        query (str): The query to search for
+    """
+    url = "https://google.serper.dev/search"
+
+    payload = json.dumps({"q": query})
+    headers = {"X-API-KEY": api_key, "Content-Type": "application/json"}
+
+    response = requests.request("POST", url, headers=headers, data=payload)
+    response.raise_for_status()
+
+    json_response = response.json()
+    results = sorted(
+        json_response.get("organic", []), key=lambda x: x.get("position", 0)
+    )
+    return [
+        SearchResult(
+            link=result["link"],
+            title=result.get("title"),
+            snippet=result.get("description"),
+        )
+        for result in results[:RAG_WEB_SEARCH_RESULT_COUNT]
+    ]

backend/apps/rag/search/serpstack.py

+import json
+import logging
+
+import requests
+
+from apps.rag.search.main import SearchResult
+from config import SRC_LOG_LEVELS, RAG_WEB_SEARCH_RESULT_COUNT
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_serpstack(
+    api_key: str, query: str, https_enabled: bool = True
+) -> list[SearchResult]:
+    """Search using serpstack.com's and return the results as a list of SearchResult objects.
+
+    Args:
+        api_key (str): A serpstack.com API key
+        query (str): The query to search for
+        https_enabled (bool): Whether to use HTTPS or HTTP for the API request
+    """
+    url = f"{'https' if https_enabled else 'http'}://api.serpstack.com/search"
+
+    headers = {"Content-Type": "application/json"}
+    params = {
+        "access_key": api_key,
+        "query": query,
+    }
+
+    response = requests.request("POST", url, headers=headers, params=params)
+    response.raise_for_status()
+
+    json_response = response.json()
+    results = sorted(
+        json_response.get("organic_results", []), key=lambda x: x.get("position", 0)
+    )
+    return [
+        SearchResult(
+            link=result["url"], title=result.get("title"), snippet=result.get("snippet")
+        )
+        for result in results[:RAG_WEB_SEARCH_RESULT_COUNT]
+    ]

backend/apps/rag/search/testdata/searxng.json

+{
+	"query": "python",
+	"number_of_results": 116000000,
+	"results": [
+		{
+			"url": "https://www.python.org/",
+			"title": "Welcome to Python.org",
+			"content": "Python is a versatile and powerful language that lets you work quickly and integrate systems more effectively. Learn how to get started, download the latest version, access documentation, find jobs, and join the Python community.",
+			"engine": "bing",
+			"parsed_url": ["https", "www.python.org", "/", "", "", ""],
+			"template": "default.html",
+			"engines": ["bing", "qwant", "duckduckgo"],
+			"positions": [1, 1, 1],
+			"score": 9.0,
+			"category": "general"
+		},
+		{
+			"url": "https://wiki.nerdvpn.de/wiki/Python_(programming_language)",
+			"title": "Python (programming language) - Wikipedia",
+			"content": "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected. It supports multiple programming paradigms, including structured (particularly procedural), object-oriented and functional programming.",
+			"engine": "bing",
+			"parsed_url": ["https", "wiki.nerdvpn.de", "/wiki/Python_(programming_language)", "", "", ""],
+			"template": "default.html",
+			"engines": ["bing", "qwant", "duckduckgo"],
+			"positions": [4, 3, 2],
+			"score": 3.25,
+			"category": "general"
+		},
+		{
+			"url": "https://docs.python.org/3/tutorial/index.html",
+			"title": "The Python Tutorial \u2014 Python 3.12.3 documentation",
+			"content": "3 days ago \u00b7 Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python\u2019s elegant syntax and dynamic typing, together with its interpreted nature, make it an ideal language for scripting and rapid application development in many \u2026",
+			"engine": "bing",
+			"parsed_url": ["https", "docs.python.org", "/3/tutorial/index.html", "", "", ""],
+			"template": "default.html",
+			"engines": ["bing", "qwant", "duckduckgo"],
+			"positions": [5, 5, 3],
+			"score": 2.2,
+			"category": "general"
+		},
+		{
+			"url": "https://www.python.org/downloads/",
+			"title": "Download Python | Python.org",
+			"content": "Python is a popular programming language for various purposes. Find the latest version of Python for different operating systems, download release notes, and learn about the development process.",
+			"engine": "bing",
+			"parsed_url": ["https", "www.python.org", "/downloads/", "", "", ""],
+			"template": "default.html",
+			"engines": ["bing", "duckduckgo"],
+			"positions": [2, 2],
+			"score": 2.0,
+			"category": "general"
+		},
+		{
+			"url": "https://www.python.org/about/gettingstarted/",
+			"title": "Python For Beginners | Python.org",
+			"content": "Learn the basics of Python, a popular and easy-to-use programming language, from installing it to using it for various purposes. Find out how to access online documentation, tutorials, books, code samples, and more resources to help you get started with Python.",
+			"engine": "bing",
+			"parsed_url": ["https", "www.python.org", "/about/gettingstarted/", "", "", ""],
+			"template": "default.html",
+			"engines": ["bing", "qwant", "duckduckgo"],
+			"positions": [9, 4, 4],
+			"score": 1.8333333333333333,
+			"category": "general"
+		},
+		{
+			"url": "https://www.python.org/shell/",
+			"title": "Welcome to Python.org",
+			"content": "Python is a versatile and easy-to-use programming language that lets you work quickly. Learn more about Python, download the latest version, access documentation, find jobs, and join the community.",
+			"engine": "bing",
+			"parsed_url": ["https", "www.python.org", "/shell/", "", "", ""],
+			"template": "default.html",
+			"engines": ["bing", "qwant", "duckduckgo"],
+			"positions": [3, 10, 8],
+			"score": 1.675,
+			"category": "general"
+		},
+		{
+			"url": "https://realpython.com/",
+			"title": "Python Tutorials \u2013 Real Python",
+			"content": "Real Python offers comprehensive and up-to-date tutorials, books, and courses for Python developers of all skill levels. Whether you want to learn Python basics, web development, data science, machine learning, or more, you can find clear and practical guides and code examples here.",
+			"engine": "bing",
+			"parsed_url": ["https", "realpython.com", "/", "", "", ""],
+			"template": "default.html",
+			"engines": ["bing", "qwant", "duckduckgo"],
+			"positions": [6, 6, 5],
+			"score": 1.6,
+			"category": "general"
+		},
+		{
+			"url": "https://wiki.nerdvpn.de/wiki/Python",
+			"title": "Python",
+			"content": "Topics referred to by the same term",
+			"engine": "wikipedia",
+			"parsed_url": ["https", "wiki.nerdvpn.de", "/wiki/Python", "", "", ""],
+			"template": "default.html",
+			"engines": ["wikipedia"],
+			"positions": [1],
+			"score": 1.0,
+			"category": "general"
+		},
+		{
+			"title": "Online Python - IDE, Editor, Compiler, Interpreter",
+			"content": "Online Python IDE is a free online tool that lets you write, execute, and share Python code in the web browser. Learn about Python, its features, and its popularity as a general-purpose programming language for web development, data science, and more.",
+			"url": "https://www.online-python.com/",
+			"engine": "duckduckgo",
+			"parsed_url": ["https", "www.online-python.com", "/", "", "", ""],
+			"template": "default.html",
+			"engines": ["qwant", "duckduckgo"],
+			"positions": [8, 6],
+			"score": 0.5833333333333333,
+			"category": "general"
+		},
+		{
+			"url": "https://micropython.org/",
+			"title": "MicroPython - Python for microcontrollers",
+			"content": "MicroPython is a full Python compiler and runtime that runs on the bare-metal. You get an interactive prompt (the REPL) to execute commands immediately, along ...",
+			"img_src": null,
+			"engine": "google",
+			"parsed_url": ["https", "micropython.org", "/", "", "", ""],
+			"template": "default.html",
+			"engines": ["google"],
+			"positions": [1],
+			"score": 1.0,
+			"category": "general"
+		},
+		{
+			"url": "https://dictionary.cambridge.org/uk/dictionary/english/python",
+			"title": "PYTHON | \u0417\u043d\u0430\u0447\u0435\u043d\u043d\u044f \u0432 \u0430\u043d\u0433\u043b\u0456\u0439\u0441\u044c\u043a\u0456\u0439 \u043c\u043e\u0432\u0456 - Cambridge Dictionary",
+			"content": "Apr 17, 2024 \u2014 \u0412\u0438\u0437\u043d\u0430\u0447\u0435\u043d\u043d\u044f PYTHON: 1. a very large snake that kills animals for food by wrapping itself around them and crushing them\u2026. \u0414\u0456\u0437\u043d\u0430\u0439\u0442\u0435\u0441\u044f \u0431\u0456\u043b\u044c\u0448\u0435.",
+			"img_src": null,
+			"engine": "google",
+			"parsed_url": [
+				"https",
+				"dictionary.cambridge.org",
+				"/uk/dictionary/english/python",
+				"",
+				"",
+				""
+			],
+			"template": "default.html",
+			"engines": ["google"],
+			"positions": [2],
+			"score": 0.5,
+			"category": "general"
+		},
+		{
+			"url": "https://www.codetoday.co.uk/code",
+			"title": "Web-based Python Editor (with Turtle graphics)",
+			"content": "Quick way of starting to write Python code, including drawing with Turtle, provided by CodeToday using Trinket.io Ideal for young children to start ...",
+			"img_src": null,
+			"engine": "google",
+			"parsed_url": ["https", "www.codetoday.co.uk", "/code", "", "", ""],
+			"template": "default.html",
+			"engines": ["google"],
+			"positions": [3],
+			"score": 0.3333333333333333,
+			"category": "general"
+		},
+		{
+			"url": "https://snapcraft.io/docs/python-plugin",
+			"title": "The python plugin | Snapcraft documentation",
+			"content": "The python plugin can be used by either Python 2 or Python 3 based parts using a setup.py script for building the project, or using a package published to ...",
+			"img_src": null,
+			"engine": "google",
+			"parsed_url": ["https", "snapcraft.io", "/docs/python-plugin", "", "", ""],
+			"template": "default.html",
+			"engines": ["google"],
+			"positions": [4],
+			"score": 0.25,
+			"category": "general"
+		},
+		{
+			"url": "https://www.developer-tech.com/categories/developer-languages/developer-languages-python/",
+			"title": "Latest Python Developer News",
+			"content": "Python's status as the primary language for AI and machine learning projects, from its extensive data-handling capabilities to its flexibility and ...",
+			"img_src": null,
+			"engine": "google",
+			"parsed_url": [
+				"https",
+				"www.developer-tech.com",
+				"/categories/developer-languages/developer-languages-python/",
+				"",
+				"",
+				""
+			],
+			"template": "default.html",
+			"engines": ["google"],
+			"positions": [5],
+			"score": 0.2,
+			"category": "general"
+		},
+		{
+			"url": "https://subjectguides.york.ac.uk/coding/python",
+			"title": "Coding: a Practical Guide - Python - Subject Guides",
+			"content": "Python is a coding language used for a wide range of things, including working with data, building systems and software, and even creating games.",
+			"img_src": null,
+			"engine": "google",
+			"parsed_url": ["https", "subjectguides.york.ac.uk", "/coding/python", "", "", ""],
+			"template": "default.html",
+			"engines": ["google"],
+			"positions": [6],
+			"score": 0.16666666666666666,
+			"category": "general"
+		},
+		{
+			"url": "https://hub.salford.ac.uk/psytech/python/getting-started-python/",
+			"title": "Getting Started - Python - Salford PsyTech Home - The Hub",
+			"content": "Python in itself is a very friendly programming language, when we get to grips with writing code, once you grasp the logic, it will become very intuitive.",
+			"img_src": null,
+			"engine": "google",
+			"parsed_url": [
+				"https",
+				"hub.salford.ac.uk",
+				"/psytech/python/getting-started-python/",
+				"",
+				"",
+				""
+			],
+			"template": "default.html",
+			"engines": ["google"],
+			"positions": [7],
+			"score": 0.14285714285714285,
+			"category": "general"
+		},
+		{
+			"url": "https://snapcraft.io/docs/python-apps",
+			"title": "Python apps | Snapcraft documentation",
+			"content": "Snapcraft can be used to package and distribute Python applications in a way that enables convenient installation by users. The process of creating a snap ...",
+			"img_src": null,
+			"engine": "google",
+			"parsed_url": ["https", "snapcraft.io", "/docs/python-apps", "", "", ""],
+			"template": "default.html",
+			"engines": ["google"],
+			"positions": [8],
+			"score": 0.125,
+			"category": "general"
+		},
+		{
+			"url": "https://anvil.works/",
+			"title": "Anvil | Build Web Apps with Nothing but Python",
+			"content": "Anvil is a free Python-based drag-and-drop web app builder.\u200eSign Up \u00b7 \u200eSign in \u00b7 \u200ePricing \u00b7 \u200eForum",
+			"img_src": null,
+			"engine": "google",
+			"parsed_url": ["https", "anvil.works", "/", "", "", ""],
+			"template": "default.html",
+			"engines": ["google"],
+			"positions": [9],
+			"score": 0.1111111111111111,
+			"category": "general"
+		},
+		{
+			"url": "https://docs.python.org/",
+			"title": "Python 3.12.3 documentation",
+			"content": "3 days ago \u00b7 This is the official documentation for Python 3.12.3. Documentation sections: What's new in Python 3.12? Or all \"What's new\" documents since Python 2.0. Tutorial. Start here: a tour of Python's syntax and features. Library reference. Standard library and builtins. Language reference.",
+			"engine": "bing",
+			"parsed_url": ["https", "docs.python.org", "/", "", "", ""],
+			"template": "default.html",
+			"engines": ["bing", "duckduckgo"],
+			"positions": [7, 13],
+			"score": 0.43956043956043955,
+			"category": "general"
+		},
+		{
+			"title": "How to Use Python: Your First Steps - Real Python",
+			"content": "Learn the basics of Python syntax, installation, error handling, and more in this tutorial. You'll also code your first Python program and test your knowledge with a quiz.",
+			"url": "https://realpython.com/python-first-steps/",
+			"engine": "duckduckgo",
+			"parsed_url": ["https", "realpython.com", "/python-first-steps/", "", "", ""],
+			"template": "default.html",
+			"engines": ["qwant", "duckduckgo"],
+			"positions": [14, 7],
+			"score": 0.42857142857142855,
+			"category": "general"
+		},
+		{
+			"title": "The Python Tutorial \u2014 Python 3.11.8 documentation",
+			"content": "This tutorial introduces the reader informally to the basic concepts and features of the Python language and system. It helps to have a Python interpreter handy for hands-on experience, but all examples are self-contained, so the tutorial can be read off-line as well. For a description of standard objects and modules, see The Python Standard ...",
+			"url": "https://docs.python.org/3.11/tutorial/",
+			"engine": "duckduckgo",
+			"parsed_url": ["https", "docs.python.org", "/3.11/tutorial/", "", "", ""],
+			"template": "default.html",
+			"engines": ["duckduckgo"],
+			"positions": [7],
+			"score": 0.14285714285714285,
+			"category": "general"
+		},
+		{
+			"url": "https://realpython.com/python-introduction/",
+			"title": "Introduction to Python 3 \u2013 Real Python",
+			"content": "Python programming language, including a brief history of the development of Python and reasons why you might select Python as your language of choice.",
+			"engine": "bing",
+			"parsed_url": ["https", "realpython.com", "/python-introduction/", "", "", ""],
+			"template": "default.html",
+			"engines": ["bing"],
+			"positions": [8],
+			"score": 0.125,
+			"category": "general"
+		},
+		{
+			"title": "Our Documentation | Python.org",
+			"content": "Find online or download Python's documentation, tutorials, and guides for beginners and advanced users. Learn how to port from Python 2 to Python 3, contribute to Python, and access Python videos and books.",
+			"url": "https://www.python.org/doc/",
+			"engine": "duckduckgo",
+			"parsed_url": ["https", "www.python.org", "/doc/", "", "", ""],
+			"template": "default.html",
+			"engines": ["duckduckgo"],
+			"positions": [9],
+			"score": 0.1111111111111111,
+			"category": "general"
+		},
+		{
+			"title": "Welcome to Python.org",
+			"url": "http://www.get-python.org/shell/",
+			"content": "The mission of the Python Software Foundation is to promote, protect, and advance the Python programming language, and to support and facilitate the growth of a diverse and international community of Python programmers. Learn more. Become a Member Donate to the PSF.",
+			"engine": "qwant",
+			"parsed_url": ["http", "www.get-python.org", "/shell/", "", "", ""],
+			"template": "default.html",
+			"engines": ["qwant"],
+			"positions": [9],
+			"score": 0.1111111111111111,
+			"category": "general"
+		},
+		{
+			"title": "About Python\u2122 | Python.org",
+			"content": "Python is a powerful, fast, and versatile programming language that runs on various platforms and is easy to learn. Learn how to get started, explore the applications, and join the community of Python programmers and users.",
+			"url": "https://www.python.org/about/",
+			"engine": "duckduckgo",
+			"parsed_url": ["https", "www.python.org", "/about/", "", "", ""],
+			"template": "default.html",
+			"engines": ["duckduckgo"],
+			"positions": [11],
+			"score": 0.09090909090909091,
+			"category": "general"
+		},
+		{
+			"title": "Online Python Compiler (Interpreter) - Programiz",
+			"content": "Write and run Python code using this online tool. You can use Python Shell like IDLE, and take inputs from the user in our Python compiler.",
+			"url": "https://www.programiz.com/python-programming/online-compiler/",
+			"engine": "duckduckgo",
+			"parsed_url": [
+				"https",
+				"www.programiz.com",
+				"/python-programming/online-compiler/",
+				"",
+				"",
+				""
+			],
+			"template": "default.html",
+			"engines": ["duckduckgo"],
+			"positions": [12],
+			"score": 0.08333333333333333,
+			"category": "general"
+		},
+		{
+			"title": "Welcome to Python.org",
+			"content": "Python is a versatile and powerful language that lets you work quickly and integrate systems more effectively. Download the latest version, read the documentation, find jobs, events, success stories, and more on Python.org.",
+			"url": "https://www.python.org/?downloads",
+			"engine": "duckduckgo",
+			"parsed_url": ["https", "www.python.org", "/", "", "downloads", ""],
+			"template": "default.html",
+			"engines": ["duckduckgo"],
+			"positions": [15],
+			"score": 0.06666666666666667,
+			"category": "general"
+		},
+		{
+			"url": "https://www.matillion.com/blog/the-importance-of-python-and-its-growing-influence-on-data-productivty-a-matillion-perspective",
+			"title": "The Importance of Python and its Growing Influence on ...",
+			"content": "Jan 30, 2024 \u2014 The synergy of low-code functionality with Python's versatility empowers data professionals to orchestrate complex transformations seamlessly.",
+			"img_src": null,
+			"engine": "google",
+			"parsed_url": [
+				"https",
+				"www.matillion.com",
+				"/blog/the-importance-of-python-and-its-growing-influence-on-data-productivty-a-matillion-perspective",
+				"",
+				"",
+				""
+			],
+			"template": "default.html",
+			"engines": ["google"],
+			"positions": [10],
+			"score": 0.1,
+			"category": "general"
+		},
+		{
+			"title": "BeginnersGuide - Python Wiki",
+			"content": "This is the program that reads Python programs and carries out their instructions; you need it before you can do any Python programming. Mac and Linux distributions may include an outdated version of Python (Python 2), but you should install an updated one (Python 3). See BeginnersGuide/Download for instructions to download the correct version ...",
+			"url": "https://wiki.python.org/moin/BeginnersGuide",
+			"engine": "duckduckgo",
+			"parsed_url": ["https", "wiki.python.org", "/moin/BeginnersGuide", "", "", ""],
+			"template": "default.html",
+			"engines": ["duckduckgo"],
+			"positions": [16],
+			"score": 0.0625,
+			"category": "general"
+		},
+		{
+			"title": "Learn Python - Free Interactive Python Tutorial",
+			"content": "Learn Python from scratch or improve your skills with this website that offers tutorials, exercises, tests and certification. Explore topics such as basics, data science, advanced features and more with DataCamp.",
+			"url": "https://www.learnpython.org/",
+			"engine": "duckduckgo",
+			"parsed_url": ["https", "www.learnpython.org", "/", "", "", ""],
+			"template": "default.html",
+			"engines": ["duckduckgo"],
+			"positions": [17],
+			"score": 0.058823529411764705,
+			"category": "general"
+		}
+	],
+	"answers": [],
+	"corrections": [],
+	"infoboxes": [
+		{
+			"infobox": "Python",
+			"id": "https://en.wikipedia.org/wiki/Python_(programming_language)",
+			"content": "general-purpose programming language",
+			"img_src": "https://upload.wikimedia.org/wikipedia/commons/thumb/6/6f/.PY_file_recreation.png/500px-.PY_file_recreation.png",
+			"urls": [
+				{
+					"title": "Official website",
+					"url": "https://www.python.org/",
+					"official": true
+				},
+				{
+					"title": "Wikipedia (en)",
+					"url": "https://en.wikipedia.org/wiki/Python_(programming_language)"
+				},
+				{
+					"title": "Wikidata",
+					"url": "http://www.wikidata.org/entity/Q28865"
+				}
+			],
+			"attributes": [
+				{
+					"label": "Inception",
+					"value": "Wednesday, February 20, 1991",
+					"entity": "P571"
+				},
+				{
+					"label": "Developer",
+					"value": "Python Software Foundation, Guido van Rossum",
+					"entity": "P178"
+				},
+				{
+					"label": "Copyright license",
+					"value": "Python Software Foundation License",
+					"entity": "P275"
+				},
+				{
+					"label": "Programmed in",
+					"value": "C, Python",
+					"entity": "P277"
+				},
+				{
+					"label": "Software version identifier",
+					"value": "3.12.3, 3.13.0a6",
+					"entity": "P348"
+				}
+			],
+			"engine": "wikidata",
+			"engines": ["wikidata"]
+		}
+	],
+	"suggestions": [
+		"python turtle",
+		"micro python tutorial",
+		"python docs",
+		"python compiler",
+		"snapcraft python",
+		"micropython vs python",
+		"python online",
+		"python download"
+	],
+	"unresponsive_engines": []
+}

backend/apps/rag/search/testdata/serper.json

+{
+	"searchParameters": {
+		"q": "apple inc",
+		"gl": "us",
+		"hl": "en",
+		"autocorrect": true,
+		"page": 1,
+		"type": "search"
+	},
+	"knowledgeGraph": {
+		"title": "Apple",
+		"type": "Technology company",
+		"website": "http://www.apple.com/",
+		"imageUrl": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQwGQRv5TjjkycpctY66mOg_e2-npacrmjAb6_jAWhzlzkFE3OTjxyzbA&s=0",
+		"description": "Apple Inc. is an American multinational technology company specializing in consumer electronics, software and online services headquartered in Cupertino, California, United States.",
+		"descriptionSource": "Wikipedia",
+		"descriptionLink": "https://en.wikipedia.org/wiki/Apple_Inc.",
+		"attributes": {
+			"Headquarters": "Cupertino, CA",
+			"CEO": "Tim Cook (Aug 24, 2011–)",
+			"Founded": "April 1, 1976, Los Altos, CA",
+			"Sales": "1 (800) 692-7753",
+			"Products": "iPhone, Apple Watch, iPad, and more",
+			"Founders": "Steve Jobs, Steve Wozniak, and Ronald Wayne",
+			"Subsidiaries": "Apple Store, Beats Electronics, Beddit, and more"
+		}
+	},
+	"organic": [
+		{
+			"title": "Apple",
+			"link": "https://www.apple.com/",
+			"snippet": "Discover the innovative world of Apple and shop everything iPhone, iPad, Apple Watch, Mac, and Apple TV, plus explore accessories, entertainment, ...",
+			"sitelinks": [
+				{
+					"title": "Support",
+					"link": "https://support.apple.com/"
+				},
+				{
+					"title": "iPhone",
+					"link": "https://www.apple.com/iphone/"
+				},
+				{
+					"title": "Apple makes business better.",
+					"link": "https://www.apple.com/business/"
+				},
+				{
+					"title": "Mac",
+					"link": "https://www.apple.com/mac/"
+				}
+			],
+			"position": 1
+		},
+		{
+			"title": "Apple Inc. - Wikipedia",
+			"link": "https://en.wikipedia.org/wiki/Apple_Inc.",
+			"snippet": "Apple Inc. is an American multinational technology company specializing in consumer electronics, software and online services headquartered in Cupertino, ...",
+			"attributes": {
+				"Products": "AirPods; Apple Watch; iPad; iPhone; Mac",
+				"Founders": "Steve Jobs; Steve Wozniak; Ronald Wayne",
+				"Founded": "April 1, 1976; 46 years ago in Los Altos, California, U.S",
+				"Industry": "Consumer electronics; Software services; Online services"
+			},
+			"sitelinks": [
+				{
+					"title": "History",
+					"link": "https://en.wikipedia.org/wiki/History_of_Apple_Inc."
+				},
+				{
+					"title": "Timeline of Apple Inc. products",
+					"link": "https://en.wikipedia.org/wiki/Timeline_of_Apple_Inc._products"
+				},
+				{
+					"title": "List of software by Apple Inc.",
+					"link": "https://en.wikipedia.org/wiki/List_of_software_by_Apple_Inc."
+				},
+				{
+					"title": "Apple Store",
+					"link": "https://en.wikipedia.org/wiki/Apple_Store"
+				}
+			],
+			"position": 2
+		},
+		{
+			"title": "Apple Inc. | History, Products, Headquarters, & Facts | Britannica",
+			"link": "https://www.britannica.com/topic/Apple-Inc",
+			"snippet": "Apple Inc., formerly Apple Computer, Inc., American manufacturer of personal computers, smartphones, tablet computers, computer peripherals, ...",
+			"date": "Aug 31, 2022",
+			"attributes": {
+				"Related People": "Steve Jobs Steve Wozniak Jony Ive Tim Cook Angela Ahrendts",
+				"Date": "1976 - present",
+				"Areas Of Involvement": "peripheral device"
+			},
+			"position": 3
+		},
+		{
+			"title": "AAPL: Apple Inc Stock Price Quote - NASDAQ GS - Bloomberg.com",
+			"link": "https://www.bloomberg.com/quote/AAPL:US",
+			"snippet": "Stock analysis for Apple Inc (AAPL:NASDAQ GS) including stock price, stock chart, company news, key statistics, fundamentals and company profile.",
+			"position": 4
+		},
+		{
+			"title": "Apple Inc. (AAPL) Company Profile & Facts - Yahoo Finance",
+			"link": "https://finance.yahoo.com/quote/AAPL/profile/",
+			"snippet": "Apple Inc. designs, manufactures, and markets smartphones, personal computers, tablets, wearables, and accessories worldwide. It also sells various related ...",
+			"position": 5
+		},
+		{
+			"title": "AAPL | Apple Inc. Stock Price & News - WSJ",
+			"link": "https://www.wsj.com/market-data/quotes/AAPL",
+			"snippet": "Apple, Inc. engages in the design, manufacture, and sale of smartphones, personal computers, tablets, wearables and accessories, and other varieties of ...",
+			"position": 6
+		},
+		{
+			"title": "Apple Inc Company Profile - Apple Inc Overview - GlobalData",
+			"link": "https://www.globaldata.com/company-profile/apple-inc/",
+			"snippet": "Apple Inc (Apple) designs, manufactures, and markets smartphones, tablets, personal computers (PCs), portable and wearable devices. The company also offers ...",
+			"position": 7
+		},
+		{
+			"title": "Apple Inc (AAPL) Stock Price & News - Google Finance",
+			"link": "https://www.google.com/finance/quote/AAPL:NASDAQ?hl=en",
+			"snippet": "Get the latest Apple Inc (AAPL) real-time quote, historical performance, charts, and other financial information to help you make more informed trading and ...",
+			"position": 8
+		}
+	],
+	"peopleAlsoAsk": [
+		{
+			"question": "What does Apple Inc mean?",
+			"snippet": "Apple Inc., formerly Apple Computer, Inc., American manufacturer of personal\ncomputers, smartphones, tablet computers, computer peripherals, and computer\nsoftware. It was the first successful personal computer company and the\npopularizer of the graphical user interface.\nAug 31, 2022",
+			"title": "Apple Inc. | History, Products, Headquarters, & Facts | Britannica",
+			"link": "https://www.britannica.com/topic/Apple-Inc"
+		},
+		{
+			"question": "Is Apple and Apple Inc same?",
+			"snippet": "Apple was founded as Apple Computer Company on April 1, 1976, by Steve Jobs,\nSteve Wozniak and Ronald Wayne to develop and sell Wozniak's Apple I personal\ncomputer. It was incorporated by Jobs and Wozniak as Apple Computer, Inc.",
+			"title": "Apple Inc. - Wikipedia",
+			"link": "https://en.wikipedia.org/wiki/Apple_Inc."
+		},
+		{
+			"question": "Who owns Apple Inc?",
+			"snippet": "Apple Inc. is owned by two main institutional investors (Vanguard Group and\nBlackRock, Inc). While its major individual shareholders comprise people like\nArt Levinson, Tim Cook, Bruce Sewell, Al Gore, Johny Sroujli, and others.",
+			"title": "Who Owns Apple In 2022? - FourWeekMBA",
+			"link": "https://fourweekmba.com/who-owns-apple/"
+		},
+		{
+			"question": "What products does Apple Inc offer?",
+			"snippet": "APPLE FOOTER\nStore.\nMac.\niPad.\niPhone.\nWatch.\nAirPods.\nTV & Home.\nAirTag.",
+			"title": "More items...",
+			"link": "https://www.apple.com/business/"
+		}
+	],
+	"relatedSearches": [
+		{
+			"query": "Who invented the iPhone"
+		},
+		{
+			"query": "Apple Inc competitors"
+		},
+		{
+			"query": "Apple iPad"
+		},
+		{
+			"query": "iPhones"
+		},
+		{
+			"query": "Apple Inc us"
+		},
+		{
+			"query": "Apple company history"
+		},
+		{
+			"query": "Apple Store"
+		},
+		{
+			"query": "Apple customer service"
+		},
+		{
+			"query": "Apple Watch"
+		},
+		{
+			"query": "Apple Inc Industry"
+		},
+		{
+			"query": "Apple Inc registered address"
+		},
+		{
+			"query": "Apple Inc Bloomberg"
+		}
+	]
+}

backend/apps/rag/search/testdata/serpstack.json

+{
+	"request": {
+		"success": true,
+		"total_time_taken": 3.4,
+		"processed_timestamp": 1714968442,
+		"search_url": "http://www.google.com/search?q=mcdonalds\u0026gl=us\u0026hl=en\u0026safe=0\u0026num=10"
+	},
+	"search_parameters": {
+		"engine": "google",
+		"type": "web",
+		"device": "desktop",
+		"auto_location": "1",
+		"google_domain": "google.com",
+		"gl": "us",
+		"hl": "en",
+		"safe": "0",
+		"news_type": "all",
+		"exclude_autocorrected_results": "0",
+		"images_color": "any",
+		"page": "1",
+		"num": "10",
+		"output": "json",
+		"csv_fields": "search_parameters.query,organic_results.position,organic_results.title,organic_results.url,organic_results.domain",
+		"query": "mcdonalds",
+		"action": "search",
+		"access_key": "aac48e007e15c532bb94ffb34532a4b2",
+		"error": {}
+	},
+	"search_information": {
+		"total_results": 1170000000,
+		"time_taken_displayed": 0.49,
+		"detected_location": {},
+		"did_you_mean": {},
+		"no_results_for_original_query": false,
+		"showing_results_for": {}
+	},
+	"organic_results": [
+		{
+			"position": 1,
+			"title": "Our Full McDonald\u0027s Food Menu",
+			"snippet": "",
+			"prerender": false,
+			"cached_page_url": {},
+			"related_pages_url": {},
+			"url": "https://www.mcdonalds.com/us/en-us/full-menu.html",
+			"domain": "www.mcdonalds.com",
+			"displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a full-menu"
+		},
+		{
+			"position": 2,
+			"title": "McDonald\u0027s",
+			"snippet": "McDonald\u0027s is the world\u0027s largest fast food restaurant chain, serving over 69 million customers daily in over 100 countries in more than 40,000 outlets as of\u00a0...",
+			"prerender": false,
+			"cached_page_url": {},
+			"related_pages_url": {},
+			"url": "https://en.wikipedia.org/wiki/McDonald%27s",
+			"domain": "en.wikipedia.org",
+			"displayed_url": "https://en.wikipedia.org \u203a wiki \u203a McDonald\u0027s"
+		},
+		{
+			"position": 3,
+			"title": "Restaurants Near Me: Nearby McDonald\u0027s Locations",
+			"snippet": "",
+			"prerender": false,
+			"cached_page_url": {},
+			"related_pages_url": {},
+			"url": "https://www.mcdonalds.com/us/en-us/restaurant-locator.html",
+			"domain": "www.mcdonalds.com",
+			"displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a restaurant-locator"
+		},
+		{
+			"position": 4,
+			"title": "Download the McDonald\u0027s App: Deals, Promotions \u0026 ...",
+			"snippet": "Download the McDonald\u0027s app for Mobile Order \u0026 Pay, exclusive deals and coupons, menu information and special promotions.",
+			"prerender": false,
+			"cached_page_url": {},
+			"related_pages_url": {},
+			"url": "https://www.mcdonalds.com/us/en-us/download-app.html",
+			"domain": "www.mcdonalds.com",
+			"displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a download-app"
+		},
+		{
+			"position": 5,
+			"title": "McDonald\u0027s Restaurant Careers in the US",
+			"snippet": "McDonald\u0027s restaurant jobs are one-of-a-kind \u2013 just like you. Restaurants are hiring across all levels, from Crew team to Management. Apply today!",
+			"prerender": false,
+			"cached_page_url": {},
+			"related_pages_url": {},
+			"url": "https://jobs.mchire.com/",
+			"domain": "jobs.mchire.com",
+			"displayed_url": "https://jobs.mchire.com"
+		}
+	],
+	"inline_images": [
+		{
+			"image_url": "https://serpstack-assets.apilayer.net/2418910010831954152.png",
+			"title": ""
+		}
+	],
+	"local_results": [
+		{
+			"position": 1,
+			"title": "McDonald\u0027s",
+			"coordinates": {
+				"latitude": 0,
+				"longitude": 0
+			},
+			"address": "",
+			"rating": 0,
+			"reviews": 0,
+			"type": "",
+			"price": {},
+			"url": 0
+		},
+		{
+			"position": 2,
+			"title": "McDonald\u0027s",
+			"coordinates": {
+				"latitude": 0,
+				"longitude": 0
+			},
+			"address": "",
+			"rating": 0,
+			"reviews": 0,
+			"type": "",
+			"price": {},
+			"url": 0
+		},
+		{
+			"position": 3,
+			"title": "McDonald\u0027s",
+			"coordinates": {
+				"latitude": 0,
+				"longitude": 0
+			},
+			"address": "",
+			"rating": 0,
+			"reviews": 0,
+			"type": "",
+			"price": {},
+			"url": 0
+		}
+	],
+	"top_stories": [
+		{
+			"block_position": 1,
+			"title": "Menu nutrition",
+			"url": "/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=mcdonald%27s+double+quarter+pounder+with+cheese\u0026stick=H4sIAAAAAAAAAONgFuLUz9U3ME-vLDBX4tVP1zc0TCsuNE0ytjTTUs5OttJPy89P0c9NzSuNLyjKL8tMSS2yAvNS80qKMlOLF7Hq5ian5Ocl5qSoFyuk5Jcm5aQqFJYmFpWkFikU5JfmATUolGeWZCgkZ6SmFqcCAM4ilJtxAAAA\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4Qri56BAh0EAM",
+			"source": "",
+			"uploaded": "",
+			"uploaded_utc": "2024-05-06T04:07:22.082Z"
+		},
+		{
+			"block_position": 2,
+			"title": "Profiles",
+			"url": "https://www.instagram.com/McDonalds",
+			"source": "",
+			"uploaded": "",
+			"uploaded_utc": "2024-05-06T04:07:22.082Z"
+		},
+		{
+			"block_position": 3,
+			"title": "People also search for",
+			"url": "/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-L5Wg8IL2sxPFxxcDEhVbocy-LJPZIvZySijw0ho2hfZ-KtV-sSEEJ9lw7JuEkXHDnRK5y4Dm8aqbiLwugbLbslwjG3hO_gpDTFZK2VoUGZPy2nrmOBCy0G3PoOfoiEtct2GSZlUz0uufG-xP8emtNzQKQpvjkAm5Zmi57iVZueiD62upz7-x2N3dAbwtm6FkInAPRw1yR91zuT7F3lEaPblTW3LaRwCDC0bvaRCh9x4N9zHgY1OOQa_rzts2jf5WpXcuw4Y%3D\u0026q=Burger+King\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4Qs9oBKAB6BAhzEAI",
+			"source": "",
+			"uploaded": "",
+			"uploaded_utc": "2024-05-06T04:07:22.082Z"
+		}
+	],
+	"related_questions": [
+		{
+			"question": "What\u0027s a number 7 at McDonald\u0027s?What\u0027s a number 7 at McDonald\u0027s?What\u0027s a number 7 at McDonald\u0027s?",
+			"answer": "",
+			"title": "",
+			"displayed_url": ""
+		},
+		{
+			"question": "Why is McDonald\u0027s changing their name?Why is McDonald\u0027s changing their name?Why is McDonald\u0027s changing their name?",
+			"answer": "",
+			"title": "",
+			"displayed_url": ""
+		},
+		{
+			"question": "What is the oldest still running Mcdonalds?What is the oldest still running Mcdonalds?What is the oldest still running Mcdonalds?",
+			"answer": "",
+			"title": "",
+			"displayed_url": ""
+		},
+		{
+			"question": "Why is McDonald\u0027s now WcDonald\u0027s?Why is McDonald\u0027s now WcDonald\u0027s?Why is McDonald\u0027s now WcDonald\u0027s?",
+			"answer": "",
+			"title": "",
+			"displayed_url": ""
+		}
+	],
+	"knowledge_graph": {
+		"title": "",
+		"type": "Fast-food restaurant company",
+		"image_urls": ["https://serpstack-assets.apilayer.net/2418910010831954152.png"],
+		"description": "McDonald\u0027s Corporation is an American multinational fast food chain, founded in 1940 as a restaurant operated by Richard and Maurice McDonald, in San Bernardino, California, United States.",
+		"source": {
+			"name": "Wikipedia",
+			"url": "https://en.wikipedia.org/wiki/McDonald\u0027s"
+		},
+		"people_also_search_for": [],
+		"known_attributes": [
+			{
+				"attribute": "kc:/business/business_operation:founder",
+				"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Ray+Kroc\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-LxARWRdbk5SkoY2sDn5Qq7yOmqYGei6qZ7sfJhsjZXBPgjMlLbS7824rpJOm69GzqVWMdoNIZiFX2T4A2td14sZOn4a1BexZLtZXHU7NZdF6VsWbGMVuiSYtXdev7uaUjEJKumiwlqTAATTebOriYTEBuSzC\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHgQAg",
+				"name": "Founder: ",
+				"value": "Ray Kroc"
+			},
+			{
+				"attribute": "kc:/organization/organization:ceo",
+				"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chris+Kempczinski\u0026si=ACC90nwLLwns5sISZcdzuISy7t-NHozt8Cbt6G3WNQfC9ekAgKFbjdEFCDgxLbt57EDZGosYDGiZuq1AcBhA6IhTOSZxfVSySuGQ3VDwmmTA7Z93n3K3596jAuZH9VVv5h8PyvKJSuGuSsQWviJTl3eKj2UL1ZIWuDgkjyVMnC47rN7j0G9PlHRCCLdQF7VDQ1gubTiC4onXqLRBTbwAj6a--PD6Jv_NoA%3D%3D\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHUQAg",
+				"name": "CEO: ",
+				"value": "Chris Kempczinski (Nov 1, 2019\u2013)"
+			},
+			{
+				"attribute": "kc:/business/employer:revenue",
+				"link": "",
+				"name": "Revenue: ",
+				"value": "25.49\u00a0billion USD (2023)"
+			},
+			{
+				"attribute": "kc:/organization/organization:founded",
+				"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Des+Plaines\u0026si=ACC90nyvvWro6QmnyY1IfSdgk5wwjB1r8BGd_IWRjXqmKPQqm_yqLtI_DBi5PXGOtg_Z3qrzzEP6mcih1nN7h5A7v6OefnEJiC7a8dBR-v9LxlRubfyR6vlMr3fZ3TmVKWwz9FRpvZb1eYNt-RM7KIDKQlwGEIgINvzhxjUrv6uxSmceduzxd8W7Pkz71XGwxF0F8OlSzHlx\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECG4QAg",
+				"name": "Founded: ",
+				"value": "April 15, 1955, Des Plaines, IL"
+			},
+			{
+				"attribute": "kc:/organization/organization:headquarters",
+				"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chicago\u0026si=ACC90nyvvWro6QmnyY1IfSdgk5wwjB1r8BGd_IWRjXqmKPQqm-46AEJ_kJbUIEvsvEEZqteiYJvXVXs2ScRNDvFFpjfeAaW3dxtpTGCgcsf5RMdi6IdzOdtjJMN3ZaFwqZOmdi7tC6r0Mh1O9bnP3HrVDB9hH02m7aA6f70dCAfTdpOFnGxDU6wVMAI5MxWBE3wTugtUDOK-\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHYQAg",
+				"name": "Headquarters: ",
+				"value": "Chicago, IL"
+			},
+			{
+				"attribute": "kc:/organization/organization:president",
+				"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chris+Kempczinski\u0026si=ACC90nwLLwns5sISZcdzuISy7t-NHozt8Cbt6G3WNQfC9ekAgKFbjdEFCDgxLbt57EDZGosYDGiZuq1AcBhA6IhTOSZxfVSySuGQ3VDwmmTA7Z93n3K3596jAuZH9VVv5h8PyvKJSuGuSsQWviJTl3eKj2UL1ZIWuDgkjyVMnC47rN7j0G9PlHRCCLdQF7VDQ1gubTiC4onXqLRBTbwAj6a--PD6Jv_NoA%3D%3D\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHEQAg",
+				"name": "President: ",
+				"value": "Chris Kempczinski"
+			}
+		],
+		"website": "https://www.mcdonalds.com/us/en-us.html",
+		"profiles": [
+			{
+				"name": "Instagram",
+				"url": "https://www.instagram.com/McDonalds"
+			},
+			{
+				"name": "X (Twitter)",
+				"url": "https://twitter.com/McDonalds"
+			},
+			{
+				"name": "Facebook",
+				"url": "https://www.facebook.com/McDonaldsUS"
+			},
+			{
+				"name": "YouTube",
+				"url": "https://www.youtube.com/user/McDonaldsUS"
+			},
+			{
+				"name": "Pinterest",
+				"url": "https://www.pinterest.com/mcdonalds"
+			}
+		],
+		"founded": "April 15, 1955, Des Plaines, IL",
+		"headquarters": "Chicago, IL",
+		"founders": [
+			{
+				"name": "Ray Kroc",
+				"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Ray+Kroc\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-LxARWRdbk5SkoY2sDn5Qq7yOmqYGei6qZ7sfJhsjZXBPgjMlLbS7824rpJOm69GzqVWMdoNIZiFX2T4A2td14sZOn4a1BexZLtZXHU7NZdF6VsWbGMVuiSYtXdev7uaUjEJKumiwlqTAATTebOriYTEBuSzC\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHgQAg"
+			}
+		]
+	}
+}

backend/apps/rag/utils.py

@@ -19,8 +19,24 @@ from langchain.retrievers import (
 )
 
 from typing import Optional
-from config import SRC_LOG_LEVELS, CHROMA_CLIENT
 
+from apps.rag.search.brave import search_brave
+from apps.rag.search.google_pse import search_google_pse
+from apps.rag.search.main import SearchResult
+from apps.rag.search.searxng import search_searxng
+from apps.rag.search.serper import search_serper
+from apps.rag.search.serpstack import search_serpstack
+from config import (
+    SRC_LOG_LEVELS,
+    CHROMA_CLIENT,
+    SEARXNG_QUERY_URL,
+    GOOGLE_PSE_API_KEY,
+    GOOGLE_PSE_ENGINE_ID,
+    BRAVE_SEARCH_API_KEY,
+    SERPSTACK_API_KEY,
+    SERPSTACK_HTTPS,
+    SERPER_API_KEY,
+)
 
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["RAG"])
@@ -520,3 +536,31 @@ class RerankCompressor(BaseDocumentCompressor):
             )
             final_results.append(doc)
         return final_results
+
+
+def search_web(query: str) -> list[SearchResult]:
+    """Search the web using a search engine and return the results as a list of SearchResult objects.
+    Will look for a search engine API key in environment variables in the following order:
+    - SEARXNG_QUERY_URL
+    - GOOGLE_PSE_API_KEY + GOOGLE_PSE_ENGINE_ID
+    - BRAVE_SEARCH_API_KEY
+    - SERPSTACK_API_KEY
+    - SERPER_API_KEY
+
+    Args:
+        query (str): The query to search for
+    """
+
+    # TODO: add playwright to search the web
+    if SEARXNG_QUERY_URL:
+        return search_searxng(SEARXNG_QUERY_URL, query)
+    elif GOOGLE_PSE_API_KEY and GOOGLE_PSE_ENGINE_ID:
+        return search_google_pse(GOOGLE_PSE_API_KEY, GOOGLE_PSE_ENGINE_ID, query)
+    elif BRAVE_SEARCH_API_KEY:
+        return search_brave(BRAVE_SEARCH_API_KEY, query)
+    elif SERPSTACK_API_KEY:
+        return search_serpstack(SERPSTACK_API_KEY, query, https_enabled=SERPSTACK_HTTPS)
+    elif SERPER_API_KEY:
+        return search_serper(SERPER_API_KEY, query)
+    else:
+        raise Exception("No search engine API key found in environment variables")

backend/config.py

@@ -574,6 +574,7 @@ ENABLE_COMMUNITY_SHARING = PersistentConfig(
     os.environ.get("ENABLE_COMMUNITY_SHARING", "True").lower() == "true",
 )
 
+
 class BannerModel(BaseModel):
     id: str
     type: str
@@ -765,6 +766,28 @@ YOUTUBE_LOADER_LANGUAGE = PersistentConfig(
     os.getenv("YOUTUBE_LOADER_LANGUAGE", "en").split(","),
 )
 
+SEARXNG_QUERY_URL = os.getenv("SEARXNG_QUERY_URL", "")
+GOOGLE_PSE_API_KEY = os.getenv("GOOGLE_PSE_API_KEY", "")
+GOOGLE_PSE_ENGINE_ID = os.getenv("GOOGLE_PSE_ENGINE_ID", "")
+BRAVE_SEARCH_API_KEY = os.getenv("BRAVE_SEARCH_API_KEY", "")
+SERPSTACK_API_KEY = os.getenv("SERPSTACK_API_KEY", "")
+SERPSTACK_HTTPS = os.getenv("SERPSTACK_HTTPS", "True").lower() == "true"
+SERPER_API_KEY = os.getenv("SERPER_API_KEY", "")
+
+
+RAG_WEB_SEARCH_ENABLED = (
+    SEARXNG_QUERY_URL != ""
+    or (GOOGLE_PSE_API_KEY != "" and GOOGLE_PSE_ENGINE_ID != "")
+    or BRAVE_SEARCH_API_KEY != ""
+    or SERPSTACK_API_KEY != ""
+    or SERPER_API_KEY != ""
+)
+
+RAG_WEB_SEARCH_RESULT_COUNT = int(os.getenv("RAG_WEB_SEARCH_RESULT_COUNT", "3"))
+RAG_WEB_SEARCH_CONCURRENT_REQUESTS = int(
+    os.getenv("RAG_WEB_SEARCH_CONCURRENT_REQUESTS", "10")
+)
+
 ####################################
 # Transcribe
 ####################################

backend/constants.py

@@ -80,3 +80,7 @@ class ERROR_MESSAGES(str, Enum):
     INVALID_URL = (
         "Oops! The URL you provided is invalid. Please double-check and try again."
     )
+
+    WEB_SEARCH_ERROR = (
+        "Oops! Something went wrong while searching the web. Please try again later."
+    )

backend/main.py

@@ -54,6 +54,7 @@ from config import (
     SRC_LOG_LEVELS,
     WEBHOOK_URL,
     ENABLE_ADMIN_EXPORT,
+    RAG_WEB_SEARCH_ENABLED,
     AppConfig,
     WEBUI_BUILD_HASH,
 )
@@ -117,6 +118,14 @@ app.state.MODELS = {}
 
 origins = ["*"]
 
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
 
 # Custom middleware to add security headers
 # class SecurityHeadersMiddleware(BaseHTTPMiddleware):
@@ -220,15 +229,6 @@ class RAGMiddleware(BaseHTTPMiddleware):
 app.add_middleware(RAGMiddleware)
 
 
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=origins,
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-
 @app.middleware("http")
 async def check_url(request: Request, call_next):
     if len(app.state.MODELS) == 0:
@@ -365,9 +365,10 @@ async def get_app_config():
             "auth": WEBUI_AUTH,
             "auth_trusted_header": bool(webui_app.state.AUTH_TRUSTED_EMAIL_HEADER),
             "enable_signup": webui_app.state.config.ENABLE_SIGNUP,
+            "enable_web_search": RAG_WEB_SEARCH_ENABLED,
             "enable_image_generation": images_app.state.config.ENABLED,
-            "enable_admin_export": ENABLE_ADMIN_EXPORT,
             "enable_community_sharing": webui_app.state.config.ENABLE_COMMUNITY_SHARING,
+            "enable_admin_export": ENABLE_ADMIN_EXPORT,
         },
     }
 

src/lib/apis/openai/index.ts

 import { OPENAI_API_BASE_URL } from '$lib/constants';
 import { promptTemplate } from '$lib/utils';
+import { type Model, models, settings } from '$lib/stores';
 
 export const getOpenAIConfig = async (token: string = '') => {
 	let error = null;
@@ -391,3 +392,71 @@ export const generateTitle = async (
 
 	return res?.choices[0]?.message?.content.replace(/["']/g, '') ?? 'New Chat';
 };
+
+export const generateSearchQuery = async (
+	token: string = '',
+	model: string,
+	previousMessages: string[],
+	prompt: string,
+	url: string = OPENAI_API_BASE_URL
+): Promise<string | undefined> => {
+	let error = null;
+
+	// TODO: Allow users to specify the prompt
+	// Get the current date in the format "January 20, 2024"
+	const currentDate = new Intl.DateTimeFormat('en-US', {
+		year: 'numeric',
+		month: 'long',
+		day: '2-digit'
+	}).format(new Date());
+
+	const res = await fetch(`${url}/chat/completions`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			Authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			model: model,
+			// Few shot prompting
+			messages: [
+				{
+					role: 'assistant',
+					content: `You are tasked with generating web search queries. Give me an appropriate query to answer my question for google search. Answer with only the query. Today is ${currentDate}.`
+				},
+				{
+					role: 'user',
+					content: prompt
+				}
+				// {
+				// 	role: 'user',
+				// 	content:
+				// 		(previousMessages.length > 0
+				// 			? `Previous Questions:\n${previousMessages.join('\n')}\n\n`
+				// 			: '') + `Current Question: ${prompt}`
+				// }
+			],
+			stream: false,
+			// Restricting the max tokens to 30 to avoid long search queries
+			max_tokens: 30
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			console.log(err);
+			if ('detail' in err) {
+				error = err.detail;
+			}
+			return undefined;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res?.choices[0]?.message?.content.replace(/["']/g, '') ?? undefined;
+};

src/lib/apis/rag/index.ts

@@ -513,3 +513,35 @@ export const updateRerankingConfig = async (token: string, payload: RerankingMod
 
 	return res;
 };
+
+export const runWebSearch = async (
+	token: string,
+	query: string,
+	collection_name?: string
+): Promise<SearchDocument | undefined> => {
+	return await fetch(`${RAG_API_BASE_URL}/web/search`, {
+		method: 'POST',
+		headers: {
+			'Content-Type': 'application/json',
+			Authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			query,
+			collection_name: collection_name ?? ''
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			console.log(err);
+			return undefined;
+		});
+};
+
+export interface SearchDocument {
+	status: boolean;
+	collection_name: string;
+	filenames: string[];
+}

src/lib/components/chat/Chat.svelte

@@ -31,7 +31,11 @@
 		getTagsById,
 		updateChatById
 	} from '$lib/apis/chats';
-	import { generateOpenAIChatCompletion, generateTitle } from '$lib/apis/openai';
+	import {
+		generateOpenAIChatCompletion,
+		generateSearchQuery,
+		generateTitle
+	} from '$lib/apis/openai';
 
 	import MessageInput from '$lib/components/chat/MessageInput.svelte';
 	import Messages from '$lib/components/chat/Messages.svelte';
@@ -41,6 +45,7 @@
 	import { queryMemory } from '$lib/apis/memories';
 	import type { Writable } from 'svelte/store';
 	import type { i18n as i18nType } from 'i18next';
+	import { runWebSearch } from '$lib/apis/rag';
 	import Banner from '../common/Banner.svelte';
 	import { getUserSettings } from '$lib/apis/users';
 
@@ -60,6 +65,8 @@
 	let selectedModels = [''];
 	let atSelectedModel: Model | undefined;
 
+	let webSearchEnabled = false;
+
 	let chat = null;
 	let tags = [];
 
@@ -399,6 +406,10 @@
 					}
 					responseMessage.userContext = userContext;
 
+					if (webSearchEnabled) {
+						await getWebSearchResults(model.id, parentId, responseMessageId);
+					}
+
 					if (model?.owned_by === 'openai') {
 						await sendPromptOpenAI(model, prompt, responseMessageId, _chatId);
 					} else if (model) {
@@ -413,6 +424,73 @@
 		await chats.set(await getChatList(localStorage.token));
 	};
 
+	const getWebSearchResults = async (model: string, parentId: string, responseId: string) => {
+		const responseMessage = history.messages[responseId];
+
+		responseMessage.status = {
+			done: false,
+			action: 'web_search',
+			description: $i18n.t('Generating search query')
+		};
+		messages = messages;
+
+		const prompt = history.messages[parentId].content;
+		let searchQuery = prompt;
+		if (prompt.length > 100) {
+			searchQuery = await generateChatSearchQuery(model, prompt);
+			if (!searchQuery) {
+				toast.warning($i18n.t('No search query generated'));
+				responseMessage.status = {
+					...responseMessage.status,
+					done: true,
+					error: true,
+					description: 'No search query generated'
+				};
+				messages = messages;
+				return;
+			}
+		}
+
+		responseMessage.status = {
+			...responseMessage.status,
+			description: $i18n.t("Searching the web for '{{searchQuery}}'", { searchQuery })
+		};
+		messages = messages;
+
+		const results = await runWebSearch(localStorage.token, searchQuery);
+		if (results === undefined) {
+			toast.warning($i18n.t('No search results found'));
+			responseMessage.status = {
+				...responseMessage.status,
+				done: true,
+				error: true,
+				description: 'No search results found'
+			};
+			messages = messages;
+			return;
+		}
+
+		responseMessage.status = {
+			...responseMessage.status,
+			done: true,
+			description: $i18n.t('Searched {{count}} sites', { count: results.filenames.length }),
+			urls: results.filenames
+		};
+
+		if (responseMessage?.files ?? undefined === undefined) {
+			responseMessage.files = [];
+		}
+
+		responseMessage.files.push({
+			collection_name: results.collection_name,
+			name: searchQuery,
+			type: 'web_search_results',
+			urls: results.filenames
+		});
+
+		messages = messages;
+	};
+
 	const sendPromptOllama = async (model, userPrompt, responseMessageId, _chatId) => {
 		model = model.id;
 		const responseMessage = history.messages[responseMessageId];
@@ -475,7 +553,9 @@
 		const docs = messages
 			.filter((message) => message?.files ?? null)
 			.map((message) =>
-				message.files.filter((item) => item.type === 'doc' || item.type === 'collection')
+				message.files.filter((item) =>
+					['doc', 'collection', 'web_search_results'].includes(item.type)
+				)
 			)
 			.flat(1);
 
@@ -671,7 +751,9 @@
 		const docs = messages
 			.filter((message) => message?.files ?? null)
 			.map((message) =>
-				message.files.filter((item) => item.type === 'doc' || item.type === 'collection')
+				message.files.filter((item) =>
+					['doc', 'collection', 'web_search_results'].includes(item.type)
+				)
 			)
 			.flat(1);
 
@@ -907,7 +989,7 @@
 			const model = $models.filter((m) => m.id === responseMessage.model).at(0);
 
 			if (model) {
-				if (model?.external) {
+				if (model?.owned_by === 'openai') {
 					await sendPromptOpenAI(
 						model,
 						history.messages[responseMessage.parentId].content,
@@ -932,7 +1014,7 @@
 			const model = $models.find((model) => model.id === selectedModels[0]);
 
 			const titleModelId =
-				model?.external ?? false
+				model?.owned_by === 'openai' ?? false
 					? $settings?.title?.modelExternal ?? selectedModels[0]
 					: $settings?.title?.model ?? selectedModels[0];
 			const titleModel = $models.find((model) => model.id === titleModelId);
@@ -957,6 +1039,29 @@
 		}
 	};
 
+	const generateChatSearchQuery = async (modelId: string, prompt: string) => {
+		const model = $models.find((model) => model.id === modelId);
+		const taskModelId =
+			model?.owned_by === 'openai' ?? false
+				? $settings?.title?.modelExternal ?? modelId
+				: $settings?.title?.model ?? modelId;
+		const taskModel = $models.find((model) => model.id === taskModelId);
+
+		const previousMessages = messages
+			.filter((message) => message.role === 'user')
+			.map((message) => message.content);
+
+		return await generateSearchQuery(
+			localStorage.token,
+			taskModelId,
+			previousMessages,
+			prompt,
+			taskModel?.owned_by === 'openai' ?? false
+				? `${OPENAI_API_BASE_URL}`
+				: `${OLLAMA_API_BASE_URL}/v1`
+		);
+	};
+
 	const setChatTitle = async (_chatId, _title) => {
 		if (_chatId === $chatId) {
 			title = _title;
@@ -1081,6 +1186,7 @@
 		bind:files
 		bind:prompt
 		bind:autoScroll
+		bind:webSearchEnabled
 		bind:atSelectedModel
 		{selectedModels}
 		{messages}

src/lib/components/chat/MessageInput.svelte

 <script lang="ts">
 	import { toast } from 'svelte-sonner';
 	import { onMount, tick, getContext } from 'svelte';
-	import { type Model, mobile, settings, showSidebar, models } from '$lib/stores';
+	import { type Model, mobile, settings, showSidebar, models, config } from '$lib/stores';
 	import { blobToFile, calculateSHA256, findWordIndices } from '$lib/utils';
 
 	import {
@@ -20,6 +20,8 @@
 	import Models from './MessageInput/Models.svelte';
 	import Tooltip from '../common/Tooltip.svelte';
 	import XMark from '$lib/components/icons/XMark.svelte';
+	import InputMenu from './MessageInput/InputMenu.svelte';
+	import { t } from 'i18next';
 
 	const i18n = getContext('i18n');
 
@@ -46,8 +48,8 @@
 
 	export let files = [];
 
-	export let fileUploadEnabled = true;
 	export let speechRecognitionEnabled = true;
+	export let webSearchEnabled = false;
 
 	export let prompt = '';
 	export let messages = [];
@@ -776,37 +778,39 @@
 						{/if}
 
 						<div class=" flex">
-							{#if fileUploadEnabled}
-								<div class=" self-end mb-2 ml-1">
-									<Tooltip content={$i18n.t('Upload files')}>
-										<button
-											class="bg-gray-50 hover:bg-gray-100 text-gray-800 dark:bg-gray-850 dark:text-white dark:hover:bg-gray-800 transition rounded-full p-1.5"
-											type="button"
-											on:click={() => {
+							<div class=" ml-1 flex items-center">
+								<InputMenu
+									bind:webSearchEnabled
+									uploadFilesHandler={() => {
 										filesInputElement.click();
 									}}
+									onClose={async () => {
+										await tick();
+										chatTextAreaElement?.focus();
+									}}
+								>
+									<button
+										class="bg-gray-50 hover:bg-gray-100 text-gray-800 dark:bg-gray-850 dark:text-white dark:hover:bg-gray-800 transition rounded-full p-1.5 outline-none focus:outline-none"
+										type="button"
 									>
 										<svg
 											xmlns="http://www.w3.org/2000/svg"
 											viewBox="0 0 16 16"
 											fill="currentColor"
-												class="w-[1.2rem] h-[1.2rem]"
+											class="size-5"
 										>
 											<path
 												d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
 											/>
 										</svg>
 									</button>
-									</Tooltip>
+								</InputMenu>
 							</div>
-							{/if}
 
 							<textarea
 								id="chat-textarea"
 								bind:this={chatTextAreaElement}
-								class="scrollbar-hidden bg-gray-50 dark:bg-gray-850 dark:text-gray-100 outline-none w-full py-3 px-3 {fileUploadEnabled
-									? ''
-									: ' pl-4'} rounded-xl resize-none h-[48px]"
+								class="scrollbar-hidden bg-gray-50 dark:bg-gray-850 dark:text-gray-100 outline-none w-full py-3 px-3 rounded-xl resize-none h-[48px]"
 								placeholder={chatInputPlaceholder !== ''
 									? chatInputPlaceholder
 									: isRecording