Unverified Commit 9f0c9d97 authored by Timothy Jaeryang Baek's avatar Timothy Jaeryang Baek Committed by GitHub
Browse files

Merge pull request #4597 from michaelpoluektov/cleanup

refactor: search and replace-able cleanup
parents 1597e33a 0470146d
......@@ -15,7 +15,7 @@ from fastapi.responses import StreamingResponse, JSONResponse, FileResponse
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from typing import List
import uuid
import requests
import hashlib
......@@ -244,7 +244,7 @@ async def speech(request: Request, user=Depends(get_verified_user)):
res = r.json()
if "error" in res:
error_detail = f"External: {res['error']['message']}"
except:
except Exception:
error_detail = f"External: {e}"
raise HTTPException(
......@@ -299,7 +299,7 @@ async def speech(request: Request, user=Depends(get_verified_user)):
res = r.json()
if "error" in res:
error_detail = f"External: {res['error']['message']}"
except:
except Exception:
error_detail = f"External: {e}"
raise HTTPException(
......@@ -353,7 +353,7 @@ def transcribe(
try:
model = WhisperModel(**whisper_kwargs)
except:
except Exception:
log.warning(
"WhisperModel initialization failed, attempting download with local_files_only=False"
)
......@@ -421,7 +421,7 @@ def transcribe(
res = r.json()
if "error" in res:
error_detail = f"External: {res['error']['message']}"
except:
except Exception:
error_detail = f"External: {e}"
raise HTTPException(
......@@ -438,7 +438,7 @@ def transcribe(
)
def get_available_models() -> List[dict]:
def get_available_models() -> list[dict]:
if app.state.config.TTS_ENGINE == "openai":
return [{"id": "tts-1"}, {"id": "tts-1-hd"}]
elif app.state.config.TTS_ENGINE == "elevenlabs":
......@@ -466,7 +466,7 @@ async def get_models(user=Depends(get_verified_user)):
return {"models": get_available_models()}
def get_available_voices() -> List[dict]:
def get_available_voices() -> list[dict]:
if app.state.config.TTS_ENGINE == "openai":
return [
{"name": "alloy", "id": "alloy"},
......
......@@ -94,7 +94,7 @@ app.state.config.COMFYUI_FLUX_FP8_CLIP = COMFYUI_FLUX_FP8_CLIP
def get_automatic1111_api_auth():
if app.state.config.AUTOMATIC1111_API_AUTH == None:
if app.state.config.AUTOMATIC1111_API_AUTH is None:
return ""
else:
auth1111_byte_string = app.state.config.AUTOMATIC1111_API_AUTH.encode("utf-8")
......@@ -145,7 +145,7 @@ async def get_engine_url(user=Depends(get_admin_user)):
async def update_engine_url(
form_data: EngineUrlUpdateForm, user=Depends(get_admin_user)
):
if form_data.AUTOMATIC1111_BASE_URL == None:
if form_data.AUTOMATIC1111_BASE_URL is None:
app.state.config.AUTOMATIC1111_BASE_URL = AUTOMATIC1111_BASE_URL
else:
url = form_data.AUTOMATIC1111_BASE_URL.strip("/")
......@@ -156,7 +156,7 @@ async def update_engine_url(
except Exception as e:
raise HTTPException(status_code=400, detail=ERROR_MESSAGES.INVALID_URL)
if form_data.COMFYUI_BASE_URL == None:
if form_data.COMFYUI_BASE_URL is None:
app.state.config.COMFYUI_BASE_URL = COMFYUI_BASE_URL
else:
url = form_data.COMFYUI_BASE_URL.strip("/")
......@@ -168,7 +168,7 @@ async def update_engine_url(
except Exception as e:
raise HTTPException(status_code=400, detail=ERROR_MESSAGES.INVALID_URL)
if form_data.AUTOMATIC1111_API_AUTH == None:
if form_data.AUTOMATIC1111_API_AUTH is None:
app.state.config.AUTOMATIC1111_API_AUTH = AUTOMATIC1111_API_AUTH
else:
app.state.config.AUTOMATIC1111_API_AUTH = form_data.AUTOMATIC1111_API_AUTH
......
......@@ -21,7 +21,7 @@ import asyncio
import logging
import time
from urllib.parse import urlparse
from typing import Optional, List, Union
from typing import Optional, Union
from starlette.background import BackgroundTask
......@@ -114,7 +114,7 @@ async def get_ollama_api_urls(user=Depends(get_admin_user)):
class UrlUpdateForm(BaseModel):
urls: List[str]
urls: list[str]
@app.post("/urls/update")
......@@ -646,7 +646,7 @@ def generate_ollama_embeddings(
class GenerateCompletionForm(BaseModel):
model: str
prompt: str
images: Optional[List[str]] = None
images: Optional[list[str]] = None
format: Optional[str] = None
options: Optional[dict] = None
system: Optional[str] = None
......@@ -689,12 +689,12 @@ async def generate_completion(
class ChatMessage(BaseModel):
role: str
content: str
images: Optional[List[str]] = None
images: Optional[list[str]] = None
class GenerateChatCompletionForm(BaseModel):
model: str
messages: List[ChatMessage]
messages: list[ChatMessage]
format: Optional[str] = None
options: Optional[dict] = None
template: Optional[str] = None
......@@ -772,7 +772,7 @@ class OpenAIChatMessage(BaseModel):
class OpenAIChatCompletionForm(BaseModel):
model: str
messages: List[OpenAIChatMessage]
messages: list[OpenAIChatMessage]
model_config = ConfigDict(extra="allow")
......
......@@ -33,7 +33,7 @@ from config import (
MODEL_FILTER_LIST,
AppConfig,
)
from typing import List, Optional, Literal, overload
from typing import Optional, Literal, overload
import hashlib
......@@ -89,11 +89,11 @@ async def update_config(form_data: OpenAIConfigForm, user=Depends(get_admin_user
class UrlsUpdateForm(BaseModel):
urls: List[str]
urls: list[str]
class KeysUpdateForm(BaseModel):
keys: List[str]
keys: list[str]
@app.get("/urls")
......
......@@ -13,7 +13,7 @@ import os, shutil, logging, re
from datetime import datetime
from pathlib import Path
from typing import List, Union, Sequence, Iterator, Any
from typing import Union, Sequence, Iterator, Any
from chromadb.utils.batch_utils import create_batches
from langchain_core.documents import Document
......@@ -439,7 +439,7 @@ class ChunkParamUpdateForm(BaseModel):
class YoutubeLoaderConfig(BaseModel):
language: List[str]
language: list[str]
translation: Optional[str] = None
......@@ -642,7 +642,7 @@ def query_doc_handler(
class QueryCollectionsForm(BaseModel):
collection_names: List[str]
collection_names: list[str]
query: str
k: Optional[int] = None
r: Optional[float] = None
......@@ -1021,7 +1021,7 @@ class TikaLoader:
self.file_path = file_path
self.mime_type = mime_type
def load(self) -> List[Document]:
def load(self) -> list[Document]:
with open(self.file_path, "rb") as f:
data = f.read()
......@@ -1185,7 +1185,7 @@ def store_doc(
f.close()
f = open(file_path, "rb")
if collection_name == None:
if collection_name is None:
collection_name = calculate_sha256(f)[:63]
f.close()
......@@ -1238,7 +1238,7 @@ def process_doc(
f = open(file_path, "rb")
collection_name = form_data.collection_name
if collection_name == None:
if collection_name is None:
collection_name = calculate_sha256(f)[:63]
f.close()
......@@ -1296,7 +1296,7 @@ def store_text(
):
collection_name = form_data.collection_name
if collection_name == None:
if collection_name is None:
collection_name = calculate_sha256_string(form_data.content)
result = store_text_in_vector_db(
......@@ -1339,7 +1339,7 @@ def scan_docs_dir(user=Depends(get_admin_user)):
sanitized_filename = sanitize_filename(filename)
doc = Documents.get_doc_by_name(sanitized_filename)
if doc == None:
if doc is None:
doc = Documents.insert_new_doc(
user.id,
DocumentForm(
......
import logging
from typing import List, Optional
from typing import Optional
import requests
from apps.rag.search.main import SearchResult, get_filtered_results
......@@ -10,7 +10,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_brave(
api_key: str, query: str, count: int, filter_list: Optional[List[str]] = None
api_key: str, query: str, count: int, filter_list: Optional[list[str]] = None
) -> list[SearchResult]:
"""Search using Brave's Search API and return the results as a list of SearchResult objects.
......
import logging
from typing import List, Optional
from typing import Optional
from apps.rag.search.main import SearchResult, get_filtered_results
from duckduckgo_search import DDGS
from config import SRC_LOG_LEVELS
......@@ -9,7 +9,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_duckduckgo(
query: str, count: int, filter_list: Optional[List[str]] = None
query: str, count: int, filter_list: Optional[list[str]] = None
) -> list[SearchResult]:
"""
Search using DuckDuckGo's Search API and return the results as a list of SearchResult objects.
......@@ -18,7 +18,7 @@ def search_duckduckgo(
count (int): The number of results to return
Returns:
List[SearchResult]: A list of search results
list[SearchResult]: A list of search results
"""
# Use the DDGS context manager to create a DDGS object
with DDGS() as ddgs:
......
import json
import logging
from typing import List, Optional
from typing import Optional
import requests
from apps.rag.search.main import SearchResult, get_filtered_results
......@@ -15,7 +15,7 @@ def search_google_pse(
search_engine_id: str,
query: str,
count: int,
filter_list: Optional[List[str]] = None,
filter_list: Optional[list[str]] = None,
) -> list[SearchResult]:
"""Search using Google's Programmable Search Engine API and return the results as a list of SearchResult objects.
......
......@@ -17,7 +17,7 @@ def search_jina(query: str, count: int) -> list[SearchResult]:
count (int): The number of results to return
Returns:
List[SearchResult]: A list of search results
list[SearchResult]: A list of search results
"""
jina_search_endpoint = "https://s.jina.ai/"
headers = {
......
import logging
import requests
from typing import List, Optional
from typing import Optional
from apps.rag.search.main import SearchResult, get_filtered_results
from config import SRC_LOG_LEVELS
......@@ -14,9 +14,9 @@ def search_searxng(
query_url: str,
query: str,
count: int,
filter_list: Optional[List[str]] = None,
filter_list: Optional[list[str]] = None,
**kwargs,
) -> List[SearchResult]:
) -> list[SearchResult]:
"""
Search a SearXNG instance for a given query and return the results as a list of SearchResult objects.
......@@ -31,10 +31,10 @@ def search_searxng(
language (str): Language filter for the search results; e.g., "en-US". Defaults to an empty string.
safesearch (int): Safe search filter for safer web results; 0 = off, 1 = moderate, 2 = strict. Defaults to 1 (moderate).
time_range (str): Time range for filtering results by date; e.g., "2023-04-05..today" or "all-time". Defaults to ''.
categories: (Optional[List[str]]): Specific categories within which the search should be performed, defaulting to an empty string if not provided.
categories: (Optional[list[str]]): Specific categories within which the search should be performed, defaulting to an empty string if not provided.
Returns:
List[SearchResult]: A list of SearchResults sorted by relevance score in descending order.
list[SearchResult]: A list of SearchResults sorted by relevance score in descending order.
Raise:
requests.exceptions.RequestException: If a request error occurs during the search process.
......
import json
import logging
from typing import List, Optional
from typing import Optional
import requests
from apps.rag.search.main import SearchResult, get_filtered_results
......@@ -11,7 +11,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_serper(
api_key: str, query: str, count: int, filter_list: Optional[List[str]] = None
api_key: str, query: str, count: int, filter_list: Optional[list[str]] = None
) -> list[SearchResult]:
"""Search using serper.dev's API and return the results as a list of SearchResult objects.
......
import json
import logging
from typing import List, Optional
from typing import Optional
import requests
from urllib.parse import urlencode
......@@ -19,7 +19,7 @@ def search_serply(
limit: int = 10,
device_type: str = "desktop",
proxy_location: str = "US",
filter_list: Optional[List[str]] = None,
filter_list: Optional[list[str]] = None,
) -> list[SearchResult]:
"""Search using serper.dev's API and return the results as a list of SearchResult objects.
......
import json
import logging
from typing import List, Optional
from typing import Optional
import requests
from apps.rag.search.main import SearchResult, get_filtered_results
......@@ -14,7 +14,7 @@ def search_serpstack(
api_key: str,
query: str,
count: int,
filter_list: Optional[List[str]] = None,
filter_list: Optional[list[str]] = None,
https_enabled: bool = True,
) -> list[SearchResult]:
"""Search using serpstack.com's and return the results as a list of SearchResult objects.
......
......@@ -17,7 +17,7 @@ def search_tavily(api_key: str, query: str, count: int) -> list[SearchResult]:
query (str): The query to search for
Returns:
List[SearchResult]: A list of search results
list[SearchResult]: A list of search results
"""
url = "https://api.tavily.com/search"
data = {"query": query, "api_key": api_key}
......
......@@ -2,7 +2,7 @@ import os
import logging
import requests
from typing import List, Union
from typing import Union
from apps.ollama.main import (
generate_ollama_embeddings,
......@@ -142,7 +142,7 @@ def merge_and_sort_query_results(query_results, k, reverse=False):
def query_collection(
collection_names: List[str],
collection_names: list[str],
query: str,
embedding_function,
k: int,
......@@ -157,13 +157,13 @@ def query_collection(
embedding_function=embedding_function,
)
results.append(result)
except:
except Exception:
pass
return merge_and_sort_query_results(results, k=k)
def query_collection_with_hybrid_search(
collection_names: List[str],
collection_names: list[str],
query: str,
embedding_function,
k: int,
......@@ -182,7 +182,7 @@ def query_collection_with_hybrid_search(
r=r,
)
results.append(result)
except:
except Exception:
pass
return merge_and_sort_query_results(results, k=k, reverse=True)
......@@ -411,7 +411,7 @@ class ChromaRetriever(BaseRetriever):
query: str,
*,
run_manager: CallbackManagerForRetrieverRun,
) -> List[Document]:
) -> list[Document]:
query_embeddings = self.embedding_function(query)
results = self.collection.query(
......
......@@ -140,7 +140,7 @@ class AuthsTable:
return None
else:
return None
except:
except Exception:
return None
def authenticate_user_by_api_key(self, api_key: str) -> Optional[UserModel]:
......@@ -152,7 +152,7 @@ class AuthsTable:
try:
user = Users.get_user_by_api_key(api_key)
return user if user else None
except:
except Exception:
return False
def authenticate_user_by_trusted_header(self, email: str) -> Optional[UserModel]:
......@@ -163,7 +163,7 @@ class AuthsTable:
if auth:
user = Users.get_user_by_id(auth.id)
return user
except:
except Exception:
return None
def update_user_password_by_id(self, id: str, new_password: str) -> bool:
......@@ -174,7 +174,7 @@ class AuthsTable:
)
db.commit()
return True if result == 1 else False
except:
except Exception:
return False
def update_email_by_id(self, id: str, email: str) -> bool:
......@@ -183,7 +183,7 @@ class AuthsTable:
result = db.query(Auth).filter_by(id=id).update({"email": email})
db.commit()
return True if result == 1 else False
except:
except Exception:
return False
def delete_auth_by_id(self, id: str) -> bool:
......@@ -200,7 +200,7 @@ class AuthsTable:
return True
else:
return False
except:
except Exception:
return False
......
from pydantic import BaseModel, ConfigDict
from typing import List, Union, Optional
from typing import Union, Optional
import json
import uuid
......@@ -164,7 +164,7 @@ class ChatTable:
db.refresh(chat)
return self.get_chat_by_id(chat.share_id)
except:
except Exception:
return None
def delete_shared_chat_by_chat_id(self, chat_id: str) -> bool:
......@@ -175,7 +175,7 @@ class ChatTable:
db.commit()
return True
except:
except Exception:
return False
def update_chat_share_id_by_id(
......@@ -189,7 +189,7 @@ class ChatTable:
db.commit()
db.refresh(chat)
return ChatModel.model_validate(chat)
except:
except Exception:
return None
def toggle_chat_archive_by_id(self, id: str) -> Optional[ChatModel]:
......@@ -201,7 +201,7 @@ class ChatTable:
db.commit()
db.refresh(chat)
return ChatModel.model_validate(chat)
except:
except Exception:
return None
def archive_all_chats_by_user_id(self, user_id: str) -> bool:
......@@ -210,12 +210,12 @@ class ChatTable:
db.query(Chat).filter_by(user_id=user_id).update({"archived": True})
db.commit()
return True
except:
except Exception:
return False
def get_archived_chat_list_by_user_id(
self, user_id: str, skip: int = 0, limit: int = 50
) -> List[ChatModel]:
) -> list[ChatModel]:
with get_db() as db:
all_chats = (
......@@ -233,7 +233,7 @@ class ChatTable:
include_archived: bool = False,
skip: int = 0,
limit: int = 50,
) -> List[ChatModel]:
) -> list[ChatModel]:
with get_db() as db:
query = db.query(Chat).filter_by(user_id=user_id)
if not include_archived:
......@@ -251,7 +251,7 @@ class ChatTable:
include_archived: bool = False,
skip: int = 0,
limit: int = -1,
) -> List[ChatTitleIdResponse]:
) -> list[ChatTitleIdResponse]:
with get_db() as db:
query = db.query(Chat).filter_by(user_id=user_id)
if not include_archived:
......@@ -279,8 +279,8 @@ class ChatTable:
]
def get_chat_list_by_chat_ids(
self, chat_ids: List[str], skip: int = 0, limit: int = 50
) -> List[ChatModel]:
self, chat_ids: list[str], skip: int = 0, limit: int = 50
) -> list[ChatModel]:
with get_db() as db:
all_chats = (
db.query(Chat)
......@@ -297,7 +297,7 @@ class ChatTable:
chat = db.get(Chat, id)
return ChatModel.model_validate(chat)
except:
except Exception:
return None
def get_chat_by_share_id(self, id: str) -> Optional[ChatModel]:
......@@ -319,10 +319,10 @@ class ChatTable:
chat = db.query(Chat).filter_by(id=id, user_id=user_id).first()
return ChatModel.model_validate(chat)
except:
except Exception:
return None
def get_chats(self, skip: int = 0, limit: int = 50) -> List[ChatModel]:
def get_chats(self, skip: int = 0, limit: int = 50) -> list[ChatModel]:
with get_db() as db:
all_chats = (
......@@ -332,7 +332,7 @@ class ChatTable:
)
return [ChatModel.model_validate(chat) for chat in all_chats]
def get_chats_by_user_id(self, user_id: str) -> List[ChatModel]:
def get_chats_by_user_id(self, user_id: str) -> list[ChatModel]:
with get_db() as db:
all_chats = (
......@@ -342,7 +342,7 @@ class ChatTable:
)
return [ChatModel.model_validate(chat) for chat in all_chats]
def get_archived_chats_by_user_id(self, user_id: str) -> List[ChatModel]:
def get_archived_chats_by_user_id(self, user_id: str) -> list[ChatModel]:
with get_db() as db:
all_chats = (
......@@ -360,7 +360,7 @@ class ChatTable:
db.commit()
return True and self.delete_shared_chat_by_chat_id(id)
except:
except Exception:
return False
def delete_chat_by_id_and_user_id(self, id: str, user_id: str) -> bool:
......@@ -371,7 +371,7 @@ class ChatTable:
db.commit()
return True and self.delete_shared_chat_by_chat_id(id)
except:
except Exception:
return False
def delete_chats_by_user_id(self, user_id: str) -> bool:
......@@ -385,7 +385,7 @@ class ChatTable:
db.commit()
return True
except:
except Exception:
return False
def delete_shared_chats_by_user_id(self, user_id: str) -> bool:
......@@ -400,7 +400,7 @@ class ChatTable:
db.commit()
return True
except:
except Exception:
return False
......
from pydantic import BaseModel, ConfigDict
from typing import List, Optional
from typing import Optional
import time
import logging
......@@ -93,7 +93,7 @@ class DocumentsTable:
return DocumentModel.model_validate(result)
else:
return None
except:
except Exception:
return None
def get_doc_by_name(self, name: str) -> Optional[DocumentModel]:
......@@ -102,10 +102,10 @@ class DocumentsTable:
document = db.query(Document).filter_by(name=name).first()
return DocumentModel.model_validate(document) if document else None
except:
except Exception:
return None
def get_docs(self) -> List[DocumentModel]:
def get_docs(self) -> list[DocumentModel]:
with get_db() as db:
return [
......@@ -160,7 +160,7 @@ class DocumentsTable:
db.query(Document).filter_by(name=name).delete()
db.commit()
return True
except:
except Exception:
return False
......
from pydantic import BaseModel, ConfigDict
from typing import List, Union, Optional
from typing import Union, Optional
import time
import logging
......@@ -90,10 +90,10 @@ class FilesTable:
try:
file = db.get(File, id)
return FileModel.model_validate(file)
except:
except Exception:
return None
def get_files(self) -> List[FileModel]:
def get_files(self) -> list[FileModel]:
with get_db() as db:
return [FileModel.model_validate(file) for file in db.query(File).all()]
......@@ -107,7 +107,7 @@ class FilesTable:
db.commit()
return True
except:
except Exception:
return False
def delete_all_files(self) -> bool:
......@@ -119,7 +119,7 @@ class FilesTable:
db.commit()
return True
except:
except Exception:
return False
......
from pydantic import BaseModel, ConfigDict
from typing import List, Union, Optional
from typing import Union, Optional
import time
import logging
......@@ -122,10 +122,10 @@ class FunctionsTable:
function = db.get(Function, id)
return FunctionModel.model_validate(function)
except:
except Exception:
return None
def get_functions(self, active_only=False) -> List[FunctionModel]:
def get_functions(self, active_only=False) -> list[FunctionModel]:
with get_db() as db:
if active_only:
......@@ -141,7 +141,7 @@ class FunctionsTable:
def get_functions_by_type(
self, type: str, active_only=False
) -> List[FunctionModel]:
) -> list[FunctionModel]:
with get_db() as db:
if active_only:
......@@ -157,7 +157,7 @@ class FunctionsTable:
for function in db.query(Function).filter_by(type=type).all()
]
def get_global_filter_functions(self) -> List[FunctionModel]:
def get_global_filter_functions(self) -> list[FunctionModel]:
with get_db() as db:
return [
......@@ -167,7 +167,7 @@ class FunctionsTable:
.all()
]
def get_global_action_functions(self) -> List[FunctionModel]:
def get_global_action_functions(self) -> list[FunctionModel]:
with get_db() as db:
return [
FunctionModel.model_validate(function)
......@@ -198,7 +198,7 @@ class FunctionsTable:
db.commit()
db.refresh(function)
return self.get_function_by_id(id)
except:
except Exception:
return None
def get_user_valves_by_id_and_user_id(
......@@ -256,7 +256,7 @@ class FunctionsTable:
)
db.commit()
return self.get_function_by_id(id)
except:
except Exception:
return None
def deactivate_all_functions(self) -> Optional[bool]:
......@@ -271,7 +271,7 @@ class FunctionsTable:
)
db.commit()
return True
except:
except Exception:
return None
def delete_function_by_id(self, id: str) -> bool:
......@@ -281,7 +281,7 @@ class FunctionsTable:
db.commit()
return True
except:
except Exception:
return False
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment