"git@developer.sourcefind.cn:OpenDAS/dgl.git" did not exist on "a6bd96aa963075a0f3b7eee0a5d5170a0c954d30"
Commit e8630661 authored by Jeffrey Morgan's avatar Jeffrey Morgan
Browse files

clean up `privategpt` example

parent 89a92477
import os import os
from dotenv import load_dotenv
from chromadb.config import Settings from chromadb.config import Settings
load_dotenv()
# Define the folder for storing database # Define the folder for storing database
PERSIST_DIRECTORY = os.environ.get('PERSIST_DIRECTORY', 'db') PERSIST_DIRECTORY = os.environ.get('PERSIST_DIRECTORY', 'db')
......
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
import os import os
import glob import glob
from typing import List from typing import List
from dotenv import load_dotenv
from multiprocessing import Pool from multiprocessing import Pool
from tqdm import tqdm from tqdm import tqdm
...@@ -27,9 +26,6 @@ from langchain.docstore.document import Document ...@@ -27,9 +26,6 @@ from langchain.docstore.document import Document
from constants import CHROMA_SETTINGS from constants import CHROMA_SETTINGS
load_dotenv()
# Load environment variables # Load environment variables
persist_directory = os.environ.get('PERSIST_DIRECTORY', 'db') persist_directory = os.environ.get('PERSIST_DIRECTORY', 'db')
source_directory = os.environ.get('SOURCE_DIRECTORY', 'source_documents') source_directory = os.environ.get('SOURCE_DIRECTORY', 'source_documents')
...@@ -37,7 +33,6 @@ embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME', 'all-MiniLM-L6-v ...@@ -37,7 +33,6 @@ embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME', 'all-MiniLM-L6-v
chunk_size = 500 chunk_size = 500
chunk_overlap = 50 chunk_overlap = 50
# Custom document loaders # Custom document loaders
class MyElmLoader(UnstructuredEmailLoader): class MyElmLoader(UnstructuredEmailLoader):
"""Wrapper to fallback to text/plain when default does not work""" """Wrapper to fallback to text/plain when default does not work"""
......
This diff is collapsed.
#!/usr/bin/env python3 #!/usr/bin/env python3
from dotenv import load_dotenv
from langchain.chains import RetrievalQA from langchain.chains import RetrievalQA
from langchain.embeddings import HuggingFaceEmbeddings from langchain.embeddings import HuggingFaceEmbeddings
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
...@@ -9,8 +8,6 @@ import os ...@@ -9,8 +8,6 @@ import os
import argparse import argparse
import time import time
load_dotenv()
model = os.environ.get("MODEL", "llama2-uncensored") model = os.environ.get("MODEL", "llama2-uncensored")
embeddings_model_name = os.environ.get("EMBEDDINGS_MODEL_NAME", "all-MiniLM-L6-v2") embeddings_model_name = os.environ.get("EMBEDDINGS_MODEL_NAME", "all-MiniLM-L6-v2")
persist_directory = os.environ.get("PERSIST_DIRECTORY", "db") persist_directory = os.environ.get("PERSIST_DIRECTORY", "db")
......
...@@ -19,6 +19,7 @@ tabulate = "^0.9.0" ...@@ -19,6 +19,7 @@ tabulate = "^0.9.0"
pandoc = "^2.3" pandoc = "^2.3"
pypandoc = "^1.11" pypandoc = "^1.11"
tqdm = "^4.65.0" tqdm = "^4.65.0"
sentence-transformers = "^2.2.2"
[build-system] [build-system]
requires = ["poetry-core"] requires = ["poetry-core"]
......
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment