Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
open-webui
Commits
e8fc522e
"git@developer.sourcefind.cn:chenpangpang/open-webui.git" did not exist on "1e1b372d07c0c71c2d28558ce1da28bdcacb9955"
Commit
e8fc522e
authored
Jun 12, 2024
by
Timothy J. Baek
Browse files
chore: format
parent
482a41e4
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
21 additions
and
14 deletions
+21
-14
backend/apps/rag/main.py
backend/apps/rag/main.py
+11
-4
backend/apps/rag/search/serply.py
backend/apps/rag/search/serply.py
+10
-10
No files found.
backend/apps/rag/main.py
View file @
e8fc522e
...
@@ -831,7 +831,9 @@ def search_web(engine: str, query: str) -> list[SearchResult]:
...
@@ -831,7 +831,9 @@ def search_web(engine: str, query: str) -> list[SearchResult]:
@
app
.
post
(
"/web/search"
)
@
app
.
post
(
"/web/search"
)
def
store_web_search
(
form_data
:
SearchForm
,
user
=
Depends
(
get_current_user
)):
def
store_web_search
(
form_data
:
SearchForm
,
user
=
Depends
(
get_current_user
)):
try
:
try
:
logging
.
info
(
f
"trying to web search with
{
app
.
state
.
config
.
RAG_WEB_SEARCH_ENGINE
,
form_data
.
query
}
"
)
logging
.
info
(
f
"trying to web search with
{
app
.
state
.
config
.
RAG_WEB_SEARCH_ENGINE
,
form_data
.
query
}
"
)
web_results
=
search_web
(
web_results
=
search_web
(
app
.
state
.
config
.
RAG_WEB_SEARCH_ENGINE
,
form_data
.
query
app
.
state
.
config
.
RAG_WEB_SEARCH_ENGINE
,
form_data
.
query
)
)
...
@@ -1241,8 +1243,10 @@ def reset(user=Depends(get_admin_user)) -> bool:
...
@@ -1241,8 +1243,10 @@ def reset(user=Depends(get_admin_user)) -> bool:
return
True
return
True
class
SafeWebBaseLoader
(
WebBaseLoader
):
class
SafeWebBaseLoader
(
WebBaseLoader
):
"""WebBaseLoader with enhanced error handling for URLs."""
"""WebBaseLoader with enhanced error handling for URLs."""
def
lazy_load
(
self
)
->
Iterator
[
Document
]:
def
lazy_load
(
self
)
->
Iterator
[
Document
]:
"""Lazy load text from the url(s) in web_path with error handling."""
"""Lazy load text from the url(s) in web_path with error handling."""
for
path
in
self
.
web_paths
:
for
path
in
self
.
web_paths
:
...
@@ -1255,15 +1259,18 @@ class SafeWebBaseLoader(WebBaseLoader):
...
@@ -1255,15 +1259,18 @@ class SafeWebBaseLoader(WebBaseLoader):
if
title
:
=
soup
.
find
(
"title"
):
if
title
:
=
soup
.
find
(
"title"
):
metadata
[
"title"
]
=
title
.
get_text
()
metadata
[
"title"
]
=
title
.
get_text
()
if
description
:
=
soup
.
find
(
"meta"
,
attrs
=
{
"name"
:
"description"
}):
if
description
:
=
soup
.
find
(
"meta"
,
attrs
=
{
"name"
:
"description"
}):
metadata
[
"description"
]
=
description
.
get
(
"content"
,
"No description found."
)
metadata
[
"description"
]
=
description
.
get
(
"content"
,
"No description found."
)
if
html
:
=
soup
.
find
(
"html"
):
if
html
:
=
soup
.
find
(
"html"
):
metadata
[
"language"
]
=
html
.
get
(
"lang"
,
"No language found."
)
metadata
[
"language"
]
=
html
.
get
(
"lang"
,
"No language found."
)
yield
Document
(
page_content
=
text
,
metadata
=
metadata
)
yield
Document
(
page_content
=
text
,
metadata
=
metadata
)
except
Exception
as
e
:
except
Exception
as
e
:
# Log the error and continue with the next URL
# Log the error and continue with the next URL
log
.
error
(
f
"Error loading
{
path
}
:
{
e
}
"
)
log
.
error
(
f
"Error loading
{
path
}
:
{
e
}
"
)
if
ENV
==
"dev"
:
if
ENV
==
"dev"
:
@
app
.
get
(
"/ef"
)
@
app
.
get
(
"/ef"
)
...
...
backend/apps/rag/search/serply.py
View file @
e8fc522e
...
@@ -12,14 +12,14 @@ log.setLevel(SRC_LOG_LEVELS["RAG"])
...
@@ -12,14 +12,14 @@ log.setLevel(SRC_LOG_LEVELS["RAG"])
def
search_serply
(
def
search_serply
(
api_key
:
str
,
api_key
:
str
,
query
:
str
,
query
:
str
,
count
:
int
,
count
:
int
,
hl
:
str
=
"us"
,
hl
:
str
=
"us"
,
limit
:
int
=
10
,
limit
:
int
=
10
,
device_type
:
str
=
"desktop"
,
device_type
:
str
=
"desktop"
,
proxy_location
:
str
=
"US"
proxy_location
:
str
=
"US"
,
)
->
list
[
SearchResult
]:
)
->
list
[
SearchResult
]:
"""Search using serper.dev's API and return the results as a list of SearchResult objects.
"""Search using serper.dev's API and return the results as a list of SearchResult objects.
Args:
Args:
...
@@ -37,7 +37,7 @@ def search_serply(
...
@@ -37,7 +37,7 @@ def search_serply(
"language"
:
"en"
,
"language"
:
"en"
,
"num"
:
limit
,
"num"
:
limit
,
"gl"
:
proxy_location
.
upper
(),
"gl"
:
proxy_location
.
upper
(),
"hl"
:
hl
.
lower
()
"hl"
:
hl
.
lower
()
,
}
}
url
=
f
"
{
url
}{
urlencode
(
query_payload
)
}
"
url
=
f
"
{
url
}{
urlencode
(
query_payload
)
}
"
...
@@ -45,7 +45,7 @@ def search_serply(
...
@@ -45,7 +45,7 @@ def search_serply(
"X-API-KEY"
:
api_key
,
"X-API-KEY"
:
api_key
,
"X-User-Agent"
:
device_type
,
"X-User-Agent"
:
device_type
,
"User-Agent"
:
"open-webui"
,
"User-Agent"
:
"open-webui"
,
"X-Proxy-Location"
:
proxy_location
"X-Proxy-Location"
:
proxy_location
,
}
}
response
=
requests
.
request
(
"GET"
,
url
,
headers
=
headers
)
response
=
requests
.
request
(
"GET"
,
url
,
headers
=
headers
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment