feat: backend implementation of various search APIs

backend/apps/rag/search/brave.py

+import logging
+
+import requests
+
+from apps.rag.search.main import SearchResult
+from config import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_brave(api_key: str, query: str) -> list[SearchResult]:
+    """Search using Brave's Search API and return the results as a list of SearchResult objects.
+
+    Args:
+        api_key (str): A Brave Search API key
+        query (str): The query to search for
+    """
+    url = "https://api.search.brave.com/res/v1/web/search"
+    headers = {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip",
+        "X-Subscription-Token": api_key,
+    }
+    params = {"q": query, "count": 5}
+
+    response = requests.get(url, headers=headers, params=params)
+    response.raise_for_status()
+
+    json_response = response.json()
+    results = json_response.get("web", {}).get("results", [])
+    return [
+        SearchResult(
+            link=result["url"], title=result.get("title"), snippet=result.get("snippet")
+        )
+        for result in results[:5]
+    ]

backend/apps/rag/search/google_pse.py

+import json
+import logging
+
+import requests
+
+from apps.rag.search.main import SearchResult
+from config import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_google_pse(
+    api_key: str, search_engine_id: str, query: str
+) -> list[SearchResult]:
+    """Search using Google's Programmable Search Engine API and return the results as a list of SearchResult objects.
+
+    Args:
+        api_key (str): A Programmable Search Engine API key
+        search_engine_id (str): A Programmable Search Engine ID
+        query (str): The query to search for
+    """
+    url = "https://www.googleapis.com/customsearch/v1"
+
+    headers = {"Content-Type": "application/json"}
+    params = {
+        "cx": search_engine_id,
+        "q": query,
+        "key": api_key,
+        "num": 5,
+    }
+
+    response = requests.request("POST", url, headers=headers, params=params)
+    response.raise_for_status()
+
+    json_response = response.json()
+    results = json_response.get("items", [])
+    return [
+        SearchResult(
+            link=result["url"], title=result.get("title"), snippet=result.get("snippet")
+        )
+        for result in results
+    ]

backend/apps/rag/search/main.py

+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class SearchResult(BaseModel):
+    link: str
+    title: Optional[str]
+    snippet: Optional[str]

backend/apps/rag/search/searxng.py

+import logging
+
+import requests
+
+from apps.rag.search.main import SearchResult
+from config import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_searxng(query_url: str, query: str) -> list[SearchResult]:
+    """Search a SearXNG instance for a query and return the results as a list of SearchResult objects.
+
+    Args:
+        query_url (str): The URL of the SearXNG instance to search. Must contain "<query>" as a placeholder
+        query (str): The query to search for
+    """
+    url = query_url.replace("<query>", query)
+    if "&format=json" not in url:
+        url += "&format=json"
+    log.debug(f"searching {url}")
+
+    r = requests.get(
+        url,
+        headers={
+            "User-Agent": "Open WebUI (https://github.com/open-webui/open-webui) RAG Bot",
+            "Accept": "text/html",
+            "Accept-Encoding": "gzip, deflate",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Connection": "keep-alive",
+        },
+    )
+    r.raise_for_status()
+
+    json_response = r.json()
+    results = json_response.get("results", [])
+    sorted_results = sorted(results, key=lambda x: x.get("score", 0), reverse=True)
+    return [
+        SearchResult(
+            link=result["url"], title=result.get("title"), snippet=result.get("content")
+        )
+        for result in sorted_results[:5]
+    ]

backend/apps/rag/search/serper.py

+import json
+import logging
+
+import requests
+
+from apps.rag.search.main import SearchResult
+from config import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_serper(api_key: str, query: str) -> list[SearchResult]:
+    """Search using serper.dev's API and return the results as a list of SearchResult objects.
+
+    Args:
+        api_key (str): A serper.dev API key
+        query (str): The query to search for
+    """
+    url = "https://google.serper.dev/search"
+
+    payload = json.dumps({"q": query})
+    headers = {"X-API-KEY": api_key, "Content-Type": "application/json"}
+
+    response = requests.request("POST", url, headers=headers, data=payload)
+    response.raise_for_status()
+
+    json_response = response.json()
+    results = sorted(
+        json_response.get("organic", []), key=lambda x: x.get("position", 0)
+    )
+    return [
+        SearchResult(
+            link=result["url"],
+            title=result.get("title"),
+            snippet=result.get("description"),
+        )
+        for result in results
+    ]

backend/apps/rag/search/serpstack.py

+import json
+import logging
+
+import requests
+
+from apps.rag.search.main import SearchResult
+from config import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_serpstack(
+    api_key: str, query: str, https_enabled: bool = True
+) -> list[SearchResult]:
+    """Search using serpstack.com's and return the results as a list of SearchResult objects.
+
+    Args:
+        api_key (str): A serpstack.com API key
+        query (str): The query to search for
+        https_enabled (bool): Whether to use HTTPS or HTTP for the API request
+    """
+    url = f"{'https' if https_enabled else 'http'}://api.serpstack.com/search"
+
+    headers = {"Content-Type": "application/json"}
+    params = {
+        "access_key": api_key,
+        "query": query,
+    }
+
+    response = requests.request("POST", url, headers=headers, params=params)
+    response.raise_for_status()
+
+    json_response = response.json()
+    results = sorted(
+        json_response.get("organic_results", []), key=lambda x: x.get("position", 0)
+    )
+    return [
+        SearchResult(
+            link=result["url"], title=result.get("title"), snippet=result.get("snippet")
+        )
+        for result in results[:5]
+    ]

backend/apps/rag/search/testdata/serper.json

+{
+  "searchParameters": {
+    "q": "apple inc",
+    "gl": "us",
+    "hl": "en",
+    "autocorrect": true,
+    "page": 1,
+    "type": "search"
+  },
+  "knowledgeGraph": {
+    "title": "Apple",
+    "type": "Technology company",
+    "website": "http://www.apple.com/",
+    "imageUrl": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQwGQRv5TjjkycpctY66mOg_e2-npacrmjAb6_jAWhzlzkFE3OTjxyzbA&s=0",
+    "description": "Apple Inc. is an American multinational technology company specializing in consumer electronics, software and online services headquartered in Cupertino, California, United States.",
+    "descriptionSource": "Wikipedia",
+    "descriptionLink": "https://en.wikipedia.org/wiki/Apple_Inc.",
+    "attributes": {
+      "Headquarters": "Cupertino, CA",
+      "CEO": "Tim Cook (Aug 24, 2011–)",
+      "Founded": "April 1, 1976, Los Altos, CA",
+      "Sales": "1 (800) 692-7753",
+      "Products": "iPhone, Apple Watch, iPad, and more",
+      "Founders": "Steve Jobs, Steve Wozniak, and Ronald Wayne",
+      "Subsidiaries": "Apple Store, Beats Electronics, Beddit, and more"
+    }
+  },
+  "organic": [
+    {
+      "title": "Apple",
+      "link": "https://www.apple.com/",
+      "snippet": "Discover the innovative world of Apple and shop everything iPhone, iPad, Apple Watch, Mac, and Apple TV, plus explore accessories, entertainment, ...",
+      "sitelinks": [
+        {
+          "title": "Support",
+          "link": "https://support.apple.com/"
+        },
+        {
+          "title": "iPhone",
+          "link": "https://www.apple.com/iphone/"
+        },
+        {
+          "title": "Apple makes business better.",
+          "link": "https://www.apple.com/business/"
+        },
+        {
+          "title": "Mac",
+          "link": "https://www.apple.com/mac/"
+        }
+      ],
+      "position": 1
+    },
+    {
+      "title": "Apple Inc. - Wikipedia",
+      "link": "https://en.wikipedia.org/wiki/Apple_Inc.",
+      "snippet": "Apple Inc. is an American multinational technology company specializing in consumer electronics, software and online services headquartered in Cupertino, ...",
+      "attributes": {
+        "Products": "AirPods; Apple Watch; iPad; iPhone; Mac",
+        "Founders": "Steve Jobs; Steve Wozniak; Ronald Wayne",
+        "Founded": "April 1, 1976; 46 years ago in Los Altos, California, U.S",
+        "Industry": "Consumer electronics; Software services; Online services"
+      },
+      "sitelinks": [
+        {
+          "title": "History",
+          "link": "https://en.wikipedia.org/wiki/History_of_Apple_Inc."
+        },
+        {
+          "title": "Timeline of Apple Inc. products",
+          "link": "https://en.wikipedia.org/wiki/Timeline_of_Apple_Inc._products"
+        },
+        {
+          "title": "List of software by Apple Inc.",
+          "link": "https://en.wikipedia.org/wiki/List_of_software_by_Apple_Inc."
+        },
+        {
+          "title": "Apple Store",
+          "link": "https://en.wikipedia.org/wiki/Apple_Store"
+        }
+      ],
+      "position": 2
+    },
+    {
+      "title": "Apple Inc. | History, Products, Headquarters, & Facts | Britannica",
+      "link": "https://www.britannica.com/topic/Apple-Inc",
+      "snippet": "Apple Inc., formerly Apple Computer, Inc., American manufacturer of personal computers, smartphones, tablet computers, computer peripherals, ...",
+      "date": "Aug 31, 2022",
+      "attributes": {
+        "Related People": "Steve Jobs Steve Wozniak Jony Ive Tim Cook Angela Ahrendts",
+        "Date": "1976 - present",
+        "Areas Of Involvement": "peripheral device"
+      },
+      "position": 3
+    },
+    {
+      "title": "AAPL: Apple Inc Stock Price Quote - NASDAQ GS - Bloomberg.com",
+      "link": "https://www.bloomberg.com/quote/AAPL:US",
+      "snippet": "Stock analysis for Apple Inc (AAPL:NASDAQ GS) including stock price, stock chart, company news, key statistics, fundamentals and company profile.",
+      "position": 4
+    },
+    {
+      "title": "Apple Inc. (AAPL) Company Profile & Facts - Yahoo Finance",
+      "link": "https://finance.yahoo.com/quote/AAPL/profile/",
+      "snippet": "Apple Inc. designs, manufactures, and markets smartphones, personal computers, tablets, wearables, and accessories worldwide. It also sells various related ...",
+      "position": 5
+    },
+    {
+      "title": "AAPL | Apple Inc. Stock Price & News - WSJ",
+      "link": "https://www.wsj.com/market-data/quotes/AAPL",
+      "snippet": "Apple, Inc. engages in the design, manufacture, and sale of smartphones, personal computers, tablets, wearables and accessories, and other varieties of ...",
+      "position": 6
+    },
+    {
+      "title": "Apple Inc Company Profile - Apple Inc Overview - GlobalData",
+      "link": "https://www.globaldata.com/company-profile/apple-inc/",
+      "snippet": "Apple Inc (Apple) designs, manufactures, and markets smartphones, tablets, personal computers (PCs), portable and wearable devices. The company also offers ...",
+      "position": 7
+    },
+    {
+      "title": "Apple Inc (AAPL) Stock Price & News - Google Finance",
+      "link": "https://www.google.com/finance/quote/AAPL:NASDAQ?hl=en",
+      "snippet": "Get the latest Apple Inc (AAPL) real-time quote, historical performance, charts, and other financial information to help you make more informed trading and ...",
+      "position": 8
+    }
+  ],
+  "peopleAlsoAsk": [
+    {
+      "question": "What does Apple Inc mean?",
+      "snippet": "Apple Inc., formerly Apple Computer, Inc., American manufacturer of personal\ncomputers, smartphones, tablet computers, computer peripherals, and computer\nsoftware. It was the first successful personal computer company and the\npopularizer of the graphical user interface.\nAug 31, 2022",
+      "title": "Apple Inc. | History, Products, Headquarters, & Facts | Britannica",
+      "link": "https://www.britannica.com/topic/Apple-Inc"
+    },
+    {
+      "question": "Is Apple and Apple Inc same?",
+      "snippet": "Apple was founded as Apple Computer Company on April 1, 1976, by Steve Jobs,\nSteve Wozniak and Ronald Wayne to develop and sell Wozniak's Apple I personal\ncomputer. It was incorporated by Jobs and Wozniak as Apple Computer, Inc.",
+      "title": "Apple Inc. - Wikipedia",
+      "link": "https://en.wikipedia.org/wiki/Apple_Inc."
+    },
+    {
+      "question": "Who owns Apple Inc?",
+      "snippet": "Apple Inc. is owned by two main institutional investors (Vanguard Group and\nBlackRock, Inc). While its major individual shareholders comprise people like\nArt Levinson, Tim Cook, Bruce Sewell, Al Gore, Johny Sroujli, and others.",
+      "title": "Who Owns Apple In 2022? - FourWeekMBA",
+      "link": "https://fourweekmba.com/who-owns-apple/"
+    },
+    {
+      "question": "What products does Apple Inc offer?",
+      "snippet": "APPLE FOOTER\nStore.\nMac.\niPad.\niPhone.\nWatch.\nAirPods.\nTV & Home.\nAirTag.",
+      "title": "More items...",
+      "link": "https://www.apple.com/business/"
+    }
+  ],
+  "relatedSearches": [
+    {
+      "query": "Who invented the iPhone"
+    },
+    {
+      "query": "Apple Inc competitors"
+    },
+    {
+      "query": "Apple iPad"
+    },
+    {
+      "query": "iPhones"
+    },
+    {
+      "query": "Apple Inc us"
+    },
+    {
+      "query": "Apple company history"
+    },
+    {
+      "query": "Apple Store"
+    },
+    {
+      "query": "Apple customer service"
+    },
+    {
+      "query": "Apple Watch"
+    },
+    {
+      "query": "Apple Inc Industry"
+    },
+    {
+      "query": "Apple Inc registered address"
+    },
+    {
+      "query": "Apple Inc Bloomberg"
+    }
+  ]
+}

backend/apps/rag/search/testdata/serpstack.json

+{
+  "request": {
+    "success": true,
+    "total_time_taken": 3.4,
+    "processed_timestamp": 1714968442,
+    "search_url": "http://www.google.com/search?q=mcdonalds\u0026gl=us\u0026hl=en\u0026safe=0\u0026num=10"
+  },
+  "search_parameters": {
+    "engine": "google",
+    "type": "web",
+    "device": "desktop",
+    "auto_location": "1",
+    "google_domain": "google.com",
+    "gl": "us",
+    "hl": "en",
+    "safe": "0",
+    "news_type": "all",
+    "exclude_autocorrected_results": "0",
+    "images_color": "any",
+    "page": "1",
+    "num": "10",
+    "output": "json",
+    "csv_fields": "search_parameters.query,organic_results.position,organic_results.title,organic_results.url,organic_results.domain",
+    "query": "mcdonalds",
+    "action": "search",
+    "access_key": "aac48e007e15c532bb94ffb34532a4b2",
+    "error": {}
+  },
+  "search_information": {
+    "total_results": 1170000000,
+    "time_taken_displayed": 0.49,
+    "detected_location": {},
+    "did_you_mean": {},
+    "no_results_for_original_query": false,
+    "showing_results_for": {}
+  },
+  "organic_results": [
+    {
+      "position": 1,
+      "title": "Our Full McDonald\u0027s Food Menu",
+      "snippet": "",
+      "prerender": false,
+      "cached_page_url": {},
+      "related_pages_url": {},
+      "url": "https://www.mcdonalds.com/us/en-us/full-menu.html",
+      "domain": "www.mcdonalds.com",
+      "displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a full-menu"
+    },
+    {
+      "position": 2,
+      "title": "McDonald\u0027s",
+      "snippet": "McDonald\u0027s is the world\u0027s largest fast food restaurant chain, serving over 69 million customers daily in over 100 countries in more than 40,000 outlets as of\u00a0...",
+      "prerender": false,
+      "cached_page_url": {},
+      "related_pages_url": {},
+      "url": "https://en.wikipedia.org/wiki/McDonald%27s",
+      "domain": "en.wikipedia.org",
+      "displayed_url": "https://en.wikipedia.org \u203a wiki \u203a McDonald\u0027s"
+    },
+    {
+      "position": 3,
+      "title": "Restaurants Near Me: Nearby McDonald\u0027s Locations",
+      "snippet": "",
+      "prerender": false,
+      "cached_page_url": {},
+      "related_pages_url": {},
+      "url": "https://www.mcdonalds.com/us/en-us/restaurant-locator.html",
+      "domain": "www.mcdonalds.com",
+      "displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a restaurant-locator"
+    },
+    {
+      "position": 4,
+      "title": "Download the McDonald\u0027s App: Deals, Promotions \u0026 ...",
+      "snippet": "Download the McDonald\u0027s app for Mobile Order \u0026 Pay, exclusive deals and coupons, menu information and special promotions.",
+      "prerender": false,
+      "cached_page_url": {},
+      "related_pages_url": {},
+      "url": "https://www.mcdonalds.com/us/en-us/download-app.html",
+      "domain": "www.mcdonalds.com",
+      "displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a download-app"
+    },
+    {
+      "position": 5,
+      "title": "McDonald\u0027s Restaurant Careers in the US",
+      "snippet": "McDonald\u0027s restaurant jobs are one-of-a-kind \u2013 just like you. Restaurants are hiring across all levels, from Crew team to Management. Apply today!",
+      "prerender": false,
+      "cached_page_url": {},
+      "related_pages_url": {},
+      "url": "https://jobs.mchire.com/",
+      "domain": "jobs.mchire.com",
+      "displayed_url": "https://jobs.mchire.com"
+    }
+  ],
+  "inline_images": [
+    {
+      "image_url": "https://serpstack-assets.apilayer.net/2418910010831954152.png",
+      "title": ""
+    }
+  ],
+  "local_results": [
+    {
+      "position": 1,
+      "title": "McDonald\u0027s",
+      "coordinates": {
+        "latitude": 0,
+        "longitude": 0
+      },
+      "address": "",
+      "rating": 0,
+      "reviews": 0,
+      "type": "",
+      "price": {},
+      "url": 0
+    },
+    {
+      "position": 2,
+      "title": "McDonald\u0027s",
+      "coordinates": {
+        "latitude": 0,
+        "longitude": 0
+      },
+      "address": "",
+      "rating": 0,
+      "reviews": 0,
+      "type": "",
+      "price": {},
+      "url": 0
+    },
+    {
+      "position": 3,
+      "title": "McDonald\u0027s",
+      "coordinates": {
+        "latitude": 0,
+        "longitude": 0
+      },
+      "address": "",
+      "rating": 0,
+      "reviews": 0,
+      "type": "",
+      "price": {},
+      "url": 0
+    }
+  ],
+  "top_stories": [
+    {
+      "block_position": 1,
+      "title": "Menu nutrition",
+      "url": "/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=mcdonald%27s+double+quarter+pounder+with+cheese\u0026stick=H4sIAAAAAAAAAONgFuLUz9U3ME-vLDBX4tVP1zc0TCsuNE0ytjTTUs5OttJPy89P0c9NzSuNLyjKL8tMSS2yAvNS80qKMlOLF7Hq5ian5Ocl5qSoFyuk5Jcm5aQqFJYmFpWkFikU5JfmATUolGeWZCgkZ6SmFqcCAM4ilJtxAAAA\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4Qri56BAh0EAM",
+      "source": "",
+      "uploaded": "",
+      "uploaded_utc": "2024-05-06T04:07:22.082Z"
+    },
+    {
+      "block_position": 2,
+      "title": "Profiles",
+      "url": "https://www.instagram.com/McDonalds",
+      "source": "",
+      "uploaded": "",
+      "uploaded_utc": "2024-05-06T04:07:22.082Z"
+    },
+    {
+      "block_position": 3,
+      "title": "People also search for",
+      "url": "/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-L5Wg8IL2sxPFxxcDEhVbocy-LJPZIvZySijw0ho2hfZ-KtV-sSEEJ9lw7JuEkXHDnRK5y4Dm8aqbiLwugbLbslwjG3hO_gpDTFZK2VoUGZPy2nrmOBCy0G3PoOfoiEtct2GSZlUz0uufG-xP8emtNzQKQpvjkAm5Zmi57iVZueiD62upz7-x2N3dAbwtm6FkInAPRw1yR91zuT7F3lEaPblTW3LaRwCDC0bvaRCh9x4N9zHgY1OOQa_rzts2jf5WpXcuw4Y%3D\u0026q=Burger+King\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4Qs9oBKAB6BAhzEAI",
+      "source": "",
+      "uploaded": "",
+      "uploaded_utc": "2024-05-06T04:07:22.082Z"
+    }
+  ],
+  "related_questions": [
+    {
+      "question": "What\u0027s a number 7 at McDonald\u0027s?What\u0027s a number 7 at McDonald\u0027s?What\u0027s a number 7 at McDonald\u0027s?",
+      "answer": "",
+      "title": "",
+      "displayed_url": ""
+    },
+    {
+      "question": "Why is McDonald\u0027s changing their name?Why is McDonald\u0027s changing their name?Why is McDonald\u0027s changing their name?",
+      "answer": "",
+      "title": "",
+      "displayed_url": ""
+    },
+    {
+      "question": "What is the oldest still running Mcdonalds?What is the oldest still running Mcdonalds?What is the oldest still running Mcdonalds?",
+      "answer": "",
+      "title": "",
+      "displayed_url": ""
+    },
+    {
+      "question": "Why is McDonald\u0027s now WcDonald\u0027s?Why is McDonald\u0027s now WcDonald\u0027s?Why is McDonald\u0027s now WcDonald\u0027s?",
+      "answer": "",
+      "title": "",
+      "displayed_url": ""
+    }
+  ],
+  "knowledge_graph": {
+    "title": "",
+    "type": "Fast-food restaurant company",
+    "image_urls": [
+      "https://serpstack-assets.apilayer.net/2418910010831954152.png"
+    ],
+    "description": "McDonald\u0027s Corporation is an American multinational fast food chain, founded in 1940 as a restaurant operated by Richard and Maurice McDonald, in San Bernardino, California, United States.",
+    "source": {
+      "name": "Wikipedia",
+      "url": "https://en.wikipedia.org/wiki/McDonald\u0027s"
+    },
+    "people_also_search_for": [],
+    "known_attributes": [
+      {
+        "attribute": "kc:/business/business_operation:founder",
+        "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Ray+Kroc\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-LxARWRdbk5SkoY2sDn5Qq7yOmqYGei6qZ7sfJhsjZXBPgjMlLbS7824rpJOm69GzqVWMdoNIZiFX2T4A2td14sZOn4a1BexZLtZXHU7NZdF6VsWbGMVuiSYtXdev7uaUjEJKumiwlqTAATTebOriYTEBuSzC\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHgQAg",
+        "name": "Founder: ",
+        "value": "Ray Kroc"
+      },
+      {
+        "attribute": "kc:/organization/organization:ceo",
+        "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chris+Kempczinski\u0026si=ACC90nwLLwns5sISZcdzuISy7t-NHozt8Cbt6G3WNQfC9ekAgKFbjdEFCDgxLbt57EDZGosYDGiZuq1AcBhA6IhTOSZxfVSySuGQ3VDwmmTA7Z93n3K3596jAuZH9VVv5h8PyvKJSuGuSsQWviJTl3eKj2UL1ZIWuDgkjyVMnC47rN7j0G9PlHRCCLdQF7VDQ1gubTiC4onXqLRBTbwAj6a--PD6Jv_NoA%3D%3D\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHUQAg",
+        "name": "CEO: ",
+        "value": "Chris Kempczinski (Nov 1, 2019\u2013)"
+      },
+      {
+        "attribute": "kc:/business/employer:revenue",
+        "link": "",
+        "name": "Revenue: ",
+        "value": "25.49\u00a0billion USD (2023)"
+      },
+      {
+        "attribute": "kc:/organization/organization:founded",
+        "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Des+Plaines\u0026si=ACC90nyvvWro6QmnyY1IfSdgk5wwjB1r8BGd_IWRjXqmKPQqm_yqLtI_DBi5PXGOtg_Z3qrzzEP6mcih1nN7h5A7v6OefnEJiC7a8dBR-v9LxlRubfyR6vlMr3fZ3TmVKWwz9FRpvZb1eYNt-RM7KIDKQlwGEIgINvzhxjUrv6uxSmceduzxd8W7Pkz71XGwxF0F8OlSzHlx\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECG4QAg",
+        "name": "Founded: ",
+        "value": "April 15, 1955, Des Plaines, IL"
+      },
+      {
+        "attribute": "kc:/organization/organization:headquarters",
+        "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chicago\u0026si=ACC90nyvvWro6QmnyY1IfSdgk5wwjB1r8BGd_IWRjXqmKPQqm-46AEJ_kJbUIEvsvEEZqteiYJvXVXs2ScRNDvFFpjfeAaW3dxtpTGCgcsf5RMdi6IdzOdtjJMN3ZaFwqZOmdi7tC6r0Mh1O9bnP3HrVDB9hH02m7aA6f70dCAfTdpOFnGxDU6wVMAI5MxWBE3wTugtUDOK-\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHYQAg",
+        "name": "Headquarters: ",
+        "value": "Chicago, IL"
+      },
+      {
+        "attribute": "kc:/organization/organization:president",
+        "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chris+Kempczinski\u0026si=ACC90nwLLwns5sISZcdzuISy7t-NHozt8Cbt6G3WNQfC9ekAgKFbjdEFCDgxLbt57EDZGosYDGiZuq1AcBhA6IhTOSZxfVSySuGQ3VDwmmTA7Z93n3K3596jAuZH9VVv5h8PyvKJSuGuSsQWviJTl3eKj2UL1ZIWuDgkjyVMnC47rN7j0G9PlHRCCLdQF7VDQ1gubTiC4onXqLRBTbwAj6a--PD6Jv_NoA%3D%3D\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHEQAg",
+        "name": "President: ",
+        "value": "Chris Kempczinski"
+      }
+    ],
+    "website": "https://www.mcdonalds.com/us/en-us.html",
+    "profiles": [
+      {
+        "name": "Instagram",
+        "url": "https://www.instagram.com/McDonalds"
+      },
+      {
+        "name": "X (Twitter)",
+        "url": "https://twitter.com/McDonalds"
+      },
+      {
+        "name": "Facebook",
+        "url": "https://www.facebook.com/McDonaldsUS"
+      },
+      {
+        "name": "YouTube",
+        "url": "https://www.youtube.com/user/McDonaldsUS"
+      },
+      {
+        "name": "Pinterest",
+        "url": "https://www.pinterest.com/mcdonalds"
+      }
+    ],
+    "founded": "April 15, 1955, Des Plaines, IL",
+    "headquarters": "Chicago, IL",
+    "founders": [
+      {
+        "name": "Ray Kroc",
+        "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Ray+Kroc\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-LxARWRdbk5SkoY2sDn5Qq7yOmqYGei6qZ7sfJhsjZXBPgjMlLbS7824rpJOm69GzqVWMdoNIZiFX2T4A2td14sZOn4a1BexZLtZXHU7NZdF6VsWbGMVuiSYtXdev7uaUjEJKumiwlqTAATTebOriYTEBuSzC\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHgQAg"
+      }
+    ]
+  }
+}
\ No newline at end of file

backend/apps/rag/utils.py

@@ -19,8 +19,24 @@ from langchain.retrievers import (
 )
 
 from typing import Optional
-from config import SRC_LOG_LEVELS, CHROMA_CLIENT
 
+from apps.rag.search.brave import search_brave
+from apps.rag.search.google_pse import search_google_pse
+from apps.rag.search.main import SearchResult
+from apps.rag.search.searxng import search_searxng
+from apps.rag.search.serper import search_serper
+from apps.rag.search.serpstack import search_serpstack
+from config import (
+    SRC_LOG_LEVELS,
+    CHROMA_CLIENT,
+    SEARXNG_QUERY_URL,
+    GOOGLE_PSE_API_KEY,
+    GOOGLE_PSE_ENGINE_ID,
+    BRAVE_SEARCH_API_KEY,
+    SERPSTACK_API_KEY,
+    SERPSTACK_HTTPS,
+    SERPER_API_KEY,
+)
 
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["RAG"])
@@ -515,3 +531,35 @@ class RerankCompressor(BaseDocumentCompressor):
             )
             final_results.append(doc)
         return final_results
+
+
+def search_web(query: str) -> list[SearchResult]:
+    """Search the web using a search engine and return the results as a list of SearchResult objects.
+    Will look for a search engine API key in environment variables in the following order:
+    - SEARXNG_QUERY_URL
+    - GOOGLE_PSE_API_KEY + GOOGLE_PSE_ENGINE_ID
+    - BRAVE_SEARCH_API_KEY
+    - SERPSTACK_API_KEY
+    - SERPER_API_KEY
+
+    Args:
+        query (str): The query to search for
+    """
+    try:
+        if SEARXNG_QUERY_URL:
+            return search_searxng(SEARXNG_QUERY_URL, query)
+        elif GOOGLE_PSE_API_KEY and GOOGLE_PSE_ENGINE_ID:
+            return search_google_pse(GOOGLE_PSE_API_KEY, GOOGLE_PSE_ENGINE_ID, query)
+        elif BRAVE_SEARCH_API_KEY:
+            return search_brave(BRAVE_SEARCH_API_KEY, query)
+        elif SERPSTACK_API_KEY:
+            return search_serpstack(
+                SERPSTACK_API_KEY, query, https_enabled=SERPSTACK_HTTPS
+            )
+        elif SERPER_API_KEY:
+            return search_serper(SERPER_API_KEY, query)
+        else:
+            raise Exception("No search engine API key found in environment variables")
+    except Exception as e:
+        log.error(f"Web search failed: {e}")
+        return []

backend/config.py

@@ -533,6 +533,14 @@ RAG_OPENAI_API_KEY = os.getenv("RAG_OPENAI_API_KEY", OPENAI_API_KEY)
 
 ENABLE_LOCAL_WEB_FETCH = os.getenv("ENABLE_LOCAL_WEB_FETCH", "False").lower() == "true"
 
+SEARXNG_QUERY_URL = os.getenv("SEARXNG_QUERY_URL", "")
+GOOGLE_PSE_API_KEY = os.getenv("GOOGLE_PSE_API_KEY", "")
+GOOGLE_PSE_ENGINE_ID = os.getenv("GOOGLE_PSE_ENGINE_ID", "")
+BRAVE_SEARCH_API_KEY = os.getenv("BRAVE_SEARCH_API_KEY", "")
+SERPSTACK_API_KEY = os.getenv("SERPSTACK_API_KEY", "")
+SERPSTACK_HTTPS = os.getenv("SERPSTACK_HTTPS", "True").lower() == "true"
+SERPER_API_KEY = os.getenv("SERPER_API_KEY", "")
+
 ####################################
 # Transcribe
 ####################################