Unverified Commit c4873859 authored by Que Nguyen's avatar Que Nguyen Committed by GitHub
Browse files

Set filter_list as optional param in serpstack.py

parent bcb84235
import json
import logging
from typing import List
from typing import List, Optional
import requests
from apps.rag.search.main import SearchResult, filter_by_whitelist
from apps.rag.search.main import SearchResult, get_filtered_results
from config import SRC_LOG_LEVELS
log = logging.getLogger(__name__)
......@@ -11,7 +11,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_serpstack(
api_key: str, query: str, count: int, whitelist:List[str], https_enabled: bool = True
api_key: str, query: str, count: int, filter_list: Optional[List[str]] = None, https_enabled: bool = True
) -> list[SearchResult]:
"""Search using serpstack.com's and return the results as a list of SearchResult objects.
......@@ -35,10 +35,11 @@ def search_serpstack(
results = sorted(
json_response.get("organic_results", []), key=lambda x: x.get("position", 0)
)
filtered_results = filter_by_whitelist(results, whitelist)
if filter_list:
results = get_filtered_results(results, filter_list)
return [
SearchResult(
link=result["url"], title=result.get("title"), snippet=result.get("snippet")
)
for result in filtered_results[:count]
for result in results[:count]
]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment