Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
ollama
Commits
e8630661
Commit
e8630661
authored
Aug 11, 2023
by
Jeffrey Morgan
Browse files
clean up `privategpt` example
parent
89a92477
Changes
6
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
803 additions
and
41 deletions
+803
-41
examples/privategpt/constants.py
examples/privategpt/constants.py
+0
-3
examples/privategpt/ingest.py
examples/privategpt/ingest.py
+0
-5
examples/privategpt/poetry.lock
examples/privategpt/poetry.lock
+553
-30
examples/privategpt/privateGPT.py
examples/privategpt/privateGPT.py
+0
-3
examples/privategpt/pyproject.toml
examples/privategpt/pyproject.toml
+1
-0
examples/privategpt/requirements.txt
examples/privategpt/requirements.txt
+249
-0
No files found.
examples/privategpt/constants.py
View file @
e8630661
import
os
import
os
from
dotenv
import
load_dotenv
from
chromadb.config
import
Settings
from
chromadb.config
import
Settings
load_dotenv
()
# Define the folder for storing database
# Define the folder for storing database
PERSIST_DIRECTORY
=
os
.
environ
.
get
(
'PERSIST_DIRECTORY'
,
'db'
)
PERSIST_DIRECTORY
=
os
.
environ
.
get
(
'PERSIST_DIRECTORY'
,
'db'
)
...
...
examples/privategpt/ingest.py
View file @
e8630661
...
@@ -2,7 +2,6 @@
...
@@ -2,7 +2,6 @@
import
os
import
os
import
glob
import
glob
from
typing
import
List
from
typing
import
List
from
dotenv
import
load_dotenv
from
multiprocessing
import
Pool
from
multiprocessing
import
Pool
from
tqdm
import
tqdm
from
tqdm
import
tqdm
...
@@ -27,9 +26,6 @@ from langchain.docstore.document import Document
...
@@ -27,9 +26,6 @@ from langchain.docstore.document import Document
from
constants
import
CHROMA_SETTINGS
from
constants
import
CHROMA_SETTINGS
load_dotenv
()
# Load environment variables
# Load environment variables
persist_directory
=
os
.
environ
.
get
(
'PERSIST_DIRECTORY'
,
'db'
)
persist_directory
=
os
.
environ
.
get
(
'PERSIST_DIRECTORY'
,
'db'
)
source_directory
=
os
.
environ
.
get
(
'SOURCE_DIRECTORY'
,
'source_documents'
)
source_directory
=
os
.
environ
.
get
(
'SOURCE_DIRECTORY'
,
'source_documents'
)
...
@@ -37,7 +33,6 @@ embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME', 'all-MiniLM-L6-v
...
@@ -37,7 +33,6 @@ embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME', 'all-MiniLM-L6-v
chunk_size
=
500
chunk_size
=
500
chunk_overlap
=
50
chunk_overlap
=
50
# Custom document loaders
# Custom document loaders
class
MyElmLoader
(
UnstructuredEmailLoader
):
class
MyElmLoader
(
UnstructuredEmailLoader
):
"""Wrapper to fallback to text/plain when default does not work"""
"""Wrapper to fallback to text/plain when default does not work"""
...
...
examples/privategpt/poetry.lock
View file @
e8630661
This diff is collapsed.
Click to expand it.
examples/privategpt/privateGPT.py
View file @
e8630661
#!/usr/bin/env python3
#!/usr/bin/env python3
from
dotenv
import
load_dotenv
from
langchain.chains
import
RetrievalQA
from
langchain.chains
import
RetrievalQA
from
langchain.embeddings
import
HuggingFaceEmbeddings
from
langchain.embeddings
import
HuggingFaceEmbeddings
from
langchain.callbacks.streaming_stdout
import
StreamingStdOutCallbackHandler
from
langchain.callbacks.streaming_stdout
import
StreamingStdOutCallbackHandler
...
@@ -9,8 +8,6 @@ import os
...
@@ -9,8 +8,6 @@ import os
import
argparse
import
argparse
import
time
import
time
load_dotenv
()
model
=
os
.
environ
.
get
(
"MODEL"
,
"llama2-uncensored"
)
model
=
os
.
environ
.
get
(
"MODEL"
,
"llama2-uncensored"
)
embeddings_model_name
=
os
.
environ
.
get
(
"EMBEDDINGS_MODEL_NAME"
,
"all-MiniLM-L6-v2"
)
embeddings_model_name
=
os
.
environ
.
get
(
"EMBEDDINGS_MODEL_NAME"
,
"all-MiniLM-L6-v2"
)
persist_directory
=
os
.
environ
.
get
(
"PERSIST_DIRECTORY"
,
"db"
)
persist_directory
=
os
.
environ
.
get
(
"PERSIST_DIRECTORY"
,
"db"
)
...
...
examples/privategpt/pyproject.toml
View file @
e8630661
...
@@ -19,6 +19,7 @@ tabulate = "^0.9.0"
...
@@ -19,6 +19,7 @@ tabulate = "^0.9.0"
pandoc
=
"^2.3"
pandoc
=
"^2.3"
pypandoc
=
"^1.11"
pypandoc
=
"^1.11"
tqdm
=
"^4.65.0"
tqdm
=
"^4.65.0"
sentence-transformers
=
"^2.2.2"
[build-system]
[build-system]
requires
=
["poetry-core"]
requires
=
["poetry-core"]
...
...
examples/privategpt/requirements.txt
View file @
e8630661
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment