Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
open-webui
Commits
e8fc522e
"src/lib/vscode:/vscode.git/clone" did not exist on "66734a7080df508b0430a06ea8c21e05770e60a4"
Commit
e8fc522e
authored
Jun 12, 2024
by
Timothy J. Baek
Browse files
chore: format
parent
482a41e4
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
21 additions
and
14 deletions
+21
-14
backend/apps/rag/main.py
backend/apps/rag/main.py
+11
-4
backend/apps/rag/search/serply.py
backend/apps/rag/search/serply.py
+10
-10
No files found.
backend/apps/rag/main.py
View file @
e8fc522e
...
...
@@ -831,7 +831,9 @@ def search_web(engine: str, query: str) -> list[SearchResult]:
@
app
.
post
(
"/web/search"
)
def
store_web_search
(
form_data
:
SearchForm
,
user
=
Depends
(
get_current_user
)):
try
:
logging
.
info
(
f
"trying to web search with
{
app
.
state
.
config
.
RAG_WEB_SEARCH_ENGINE
,
form_data
.
query
}
"
)
logging
.
info
(
f
"trying to web search with
{
app
.
state
.
config
.
RAG_WEB_SEARCH_ENGINE
,
form_data
.
query
}
"
)
web_results
=
search_web
(
app
.
state
.
config
.
RAG_WEB_SEARCH_ENGINE
,
form_data
.
query
)
...
...
@@ -1241,8 +1243,10 @@ def reset(user=Depends(get_admin_user)) -> bool:
return
True
class
SafeWebBaseLoader
(
WebBaseLoader
):
"""WebBaseLoader with enhanced error handling for URLs."""
def
lazy_load
(
self
)
->
Iterator
[
Document
]:
"""Lazy load text from the url(s) in web_path with error handling."""
for
path
in
self
.
web_paths
:
...
...
@@ -1255,7 +1259,9 @@ class SafeWebBaseLoader(WebBaseLoader):
if
title
:
=
soup
.
find
(
"title"
):
metadata
[
"title"
]
=
title
.
get_text
()
if
description
:
=
soup
.
find
(
"meta"
,
attrs
=
{
"name"
:
"description"
}):
metadata
[
"description"
]
=
description
.
get
(
"content"
,
"No description found."
)
metadata
[
"description"
]
=
description
.
get
(
"content"
,
"No description found."
)
if
html
:
=
soup
.
find
(
"html"
):
metadata
[
"language"
]
=
html
.
get
(
"lang"
,
"No language found."
)
...
...
@@ -1264,6 +1270,7 @@ class SafeWebBaseLoader(WebBaseLoader):
# Log the error and continue with the next URL
log
.
error
(
f
"Error loading
{
path
}
:
{
e
}
"
)
if
ENV
==
"dev"
:
@
app
.
get
(
"/ef"
)
...
...
backend/apps/rag/search/serply.py
View file @
e8fc522e
...
...
@@ -18,8 +18,8 @@ def search_serply(
hl
:
str
=
"us"
,
limit
:
int
=
10
,
device_type
:
str
=
"desktop"
,
proxy_location
:
str
=
"US"
)
->
list
[
SearchResult
]:
proxy_location
:
str
=
"US"
,
)
->
list
[
SearchResult
]:
"""Search using serper.dev's API and return the results as a list of SearchResult objects.
Args:
...
...
@@ -37,7 +37,7 @@ def search_serply(
"language"
:
"en"
,
"num"
:
limit
,
"gl"
:
proxy_location
.
upper
(),
"hl"
:
hl
.
lower
()
"hl"
:
hl
.
lower
()
,
}
url
=
f
"
{
url
}{
urlencode
(
query_payload
)
}
"
...
...
@@ -45,7 +45,7 @@ def search_serply(
"X-API-KEY"
:
api_key
,
"X-User-Agent"
:
device_type
,
"User-Agent"
:
"open-webui"
,
"X-Proxy-Location"
:
proxy_location
"X-Proxy-Location"
:
proxy_location
,
}
response
=
requests
.
request
(
"GET"
,
url
,
headers
=
headers
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment