Skip to content

GitLab

Unverified Commit 1f02940b authored by Timothy Jaeryang Baek's avatar Timothy Jaeryang Baek Committed by GitHub
Browse files

Merge pull request #617 from ollama-webui/doc-collection 

feat: document collection
parents 3382fd10 f814b08b
Hide whitespace changes
Inline Side-by-side
Showing with 826 additions and 319 deletions
backend/apps/rag/main.py
View file @ 1f02940b
......@@ -10,6 +10,7 @@ from fastapi import (
)
from fastapi.middleware.cors import CORSMiddleware
import os, shutil
from typing import List
# from chromadb.utils import embedding_functions
......@@ -96,19 +97,22 @@ async def get_status():
return {"status": True}
@app.get("/query/{collection_name}")
def query_collection(
collection_name: str,
query: str,
k: Optional[int] = 4,
class QueryDocForm(BaseModel):
collection_name: str
query: str
k: Optional[int] = 4
@app.post("/query/doc")
def query_doc(
form_data: QueryDocForm,
user=Depends(get_current_user),
):
try:
collection = CHROMA_CLIENT.get_collection(
name=collection_name,
name=form_data.collection_name,
)
result = collection.query(query_texts=[query], n_results=k)
result = collection.query(query_texts=[form_data.query], n_results=form_data.k)
return result
except Exception as e:
print(e)
......@@ -118,6 +122,79 @@ def query_collection(
)
class QueryCollectionsForm(BaseModel):
collection_names: List[str]
query: str
k: Optional[int] = 4
def merge_and_sort_query_results(query_results, k):
# Initialize lists to store combined data
combined_ids = []
combined_distances = []
combined_metadatas = []
combined_documents = []
# Combine data from each dictionary
for data in query_results:
combined_ids.extend(data["ids"][0])
combined_distances.extend(data["distances"][0])
combined_metadatas.extend(data["metadatas"][0])
combined_documents.extend(data["documents"][0])
# Create a list of tuples (distance, id, metadata, document)
combined = list(
zip(combined_distances, combined_ids, combined_metadatas, combined_documents)
)
# Sort the list based on distances
combined.sort(key=lambda x: x[0])
# Unzip the sorted list
sorted_distances, sorted_ids, sorted_metadatas, sorted_documents = zip(*combined)
# Slicing the lists to include only k elements
sorted_distances = list(sorted_distances)[:k]
sorted_ids = list(sorted_ids)[:k]
sorted_metadatas = list(sorted_metadatas)[:k]
sorted_documents = list(sorted_documents)[:k]
# Create the output dictionary
merged_query_results = {
"ids": [sorted_ids],
"distances": [sorted_distances],
"metadatas": [sorted_metadatas],
"documents": [sorted_documents],
"embeddings": None,
"uris": None,
"data": None,
}
return merged_query_results
@app.post("/query/collection")
def query_collection(
form_data: QueryCollectionsForm,
user=Depends(get_current_user),
):
results = []
for collection_name in form_data.collection_names:
try:
collection = CHROMA_CLIENT.get_collection(
name=collection_name,
)
result = collection.query(
query_texts=[form_data.query], n_results=form_data.k
)
results.append(result)
except:
pass
return merge_and_sort_query_results(results, form_data.k)
@app.post("/web")
def store_web(form_data: StoreWebForm, user=Depends(get_current_user)):
# "https://www.gutenberg.org/files/1727/1727-h/1727-h.htm"
......
backend/apps/web/models/documents.py
View file @ 1f02940b
......@@ -44,6 +44,16 @@ class DocumentModel(BaseModel):
####################
class DocumentResponse(BaseModel):
collection_name: str
name: str
title: str
filename: str
content: Optional[dict] = None
user_id: str
timestamp: int # timestamp in epoch
class DocumentUpdateForm(BaseModel):
name: str
title: str
......@@ -111,6 +121,26 @@ class DocumentsTable:
print(e)
return None
def update_doc_content_by_name(
self, name: str, updated: dict
) -> Optional[DocumentModel]:
try:
doc = self.get_doc_by_name(name)
doc_content = json.loads(doc.content if doc.content else "{}")
doc_content = {**doc_content, **updated}
query = Document.update(
content=json.dumps(doc_content),
timestamp=int(time.time()),
).where(Document.name == name)
query.execute()
doc = Document.get(Document.name == name)
return DocumentModel(**model_to_dict(doc))
except Exception as e:
print(e)
return None
def delete_doc_by_name(self, name: str) -> bool:
try:
query = Document.delete().where((Document.name == name))
......
backend/apps/web/routers/documents.py
View file @ 1f02940b
......@@ -11,6 +11,7 @@ from apps.web.models.documents import (
DocumentForm,
DocumentUpdateForm,
DocumentModel,
DocumentResponse,
)
from utils.utils import get_current_user
......@@ -23,9 +24,18 @@ router = APIRouter()
############################
@router.get("/", response_model=List[DocumentModel])
@router.get("/", response_model=List[DocumentResponse])
async def get_documents(user=Depends(get_current_user)):
return Documents.get_docs()
docs = [
DocumentResponse(
**{
**doc.model_dump(),
"content": json.loads(doc.content if doc.content else "{}"),
}
)
for doc in Documents.get_docs()
]
return docs
############################
......@@ -33,7 +43,7 @@ async def get_documents(user=Depends(get_current_user)):
############################
@router.post("/create", response_model=Optional[DocumentModel])
@router.post("/create", response_model=Optional[DocumentResponse])
async def create_new_doc(form_data: DocumentForm, user=Depends(get_current_user)):
if user.role != "admin":
raise HTTPException(
......@@ -46,7 +56,12 @@ async def create_new_doc(form_data: DocumentForm, user=Depends(get_current_user)
doc = Documents.insert_new_doc(user.id, form_data)
if doc:
return doc
return DocumentResponse(
**{
**doc.model_dump(),
"content": json.loads(doc.content if doc.content else "{}"),
}
)
else:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
......@@ -64,12 +79,45 @@ async def create_new_doc(form_data: DocumentForm, user=Depends(get_current_user)
############################
@router.get("/name/{name}", response_model=Optional[DocumentModel])
@router.get("/name/{name}", response_model=Optional[DocumentResponse])
async def get_doc_by_name(name: str, user=Depends(get_current_user)):
doc = Documents.get_doc_by_name(name)
if doc:
return doc
return DocumentResponse(
**{
**doc.model_dump(),
"content": json.loads(doc.content if doc.content else "{}"),
}
)
else:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=ERROR_MESSAGES.NOT_FOUND,
)
############################
# TagDocByName
############################
class TagDocumentForm(BaseModel):
name: str
tags: List[dict]
@router.post("/name/{name}/tags", response_model=Optional[DocumentResponse])
async def tag_doc_by_name(form_data: TagDocumentForm, user=Depends(get_current_user)):
doc = Documents.update_doc_content_by_name(form_data.name, {"tags": form_data.tags})
if doc:
return DocumentResponse(
**{
**doc.model_dump(),
"content": json.loads(doc.content if doc.content else "{}"),
}
)
else:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
......@@ -82,7 +130,7 @@ async def get_doc_by_name(name: str, user=Depends(get_current_user)):
############################
@router.post("/name/{name}/update", response_model=Optional[DocumentModel])
@router.post("/name/{name}/update", response_model=Optional[DocumentResponse])
async def update_doc_by_name(
name: str, form_data: DocumentUpdateForm, user=Depends(get_current_user)
):
......@@ -94,7 +142,12 @@ async def update_doc_by_name(
doc = Documents.update_doc_by_name(name, form_data)
if doc:
return doc
return DocumentResponse(
**{
**doc.model_dump(),
"content": json.loads(doc.content if doc.content else "{}"),
}
)
else:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
......
src/lib/apis/documents/index.ts
View file @ 1f02940b
......@@ -144,6 +144,47 @@ export const updateDocByName = async (token: string, name: string, form: DocUpda
return res;
};
type TagDocForm = {
name: string;
tags: string[];
};
export const tagDocByName = async (token: string, name: string, form: TagDocForm) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/documents/name/${name}/tags`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify({
name: form.name,
tags: form.tags
})
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.then((json) => {
return json;
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const deleteDocByName = async (token: string, name: string) => {
let error = null;
......
src/lib/apis/rag/index.ts
View file @ 1f02940b
......@@ -64,30 +64,64 @@ export const uploadWebToVectorDB = async (token: string, collection_name: string
return res;
};
export const queryVectorDB = async (
export const queryDoc = async (
token: string,
collection_name: string,
query: string,
k: number
) => {
let error = null;
const searchParams = new URLSearchParams();
searchParams.set('query', query);
if (k) {
searchParams.set('k', k.toString());
const res = await fetch(`${RAG_API_BASE_URL}/query/doc`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify({
collection_name: collection_name,
query: query,
k: k
})
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
return null;
});
if (error) {
throw error;
}
const res = await fetch(
`${RAG_API_BASE_URL}/query/${collection_name}/?${searchParams.toString()}`,
{
method: 'GET',
headers: {
Accept: 'application/json',
authorization: `Bearer ${token}`
}
}
)
return res;
};
export const queryCollection = async (
token: string,
collection_names: string,
query: string,
k: number
) => {
let error = null;
const res = await fetch(`${RAG_API_BASE_URL}/query/collection`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify({
collection_names: collection_names,
query: query,
k: k
})
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
......
src/lib/components/AddFilesPlaceholder.svelte
View file @ 1f02940b
<div class=" text-center text-6xl mb-3">📄</div>
<div class="text-center dark:text-white text-2xl font-semibold z-50">Add Files</div>
<div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
Drop any files here to add to the conversation
</div>
<slot
><div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
Drop any files here to add to the conversation
</div>
</slot>
src/lib/components/chat/MessageInput.svelte
View file @ 1f02940b
......@@ -295,7 +295,7 @@
files = [
...files,
{
type: 'doc',
type: e?.detail?.type ?? 'doc',
...e.detail,
upload_status: true
}
......@@ -446,6 +446,34 @@
<div class=" text-gray-500 text-sm">Document</div>
</div>
</div>
{:else if file.type === 'collection'}
<div
class="h-16 w-[15rem] flex items-center space-x-3 px-2.5 dark:bg-gray-600 rounded-xl border border-gray-200 dark:border-none"
>
<div class="p-2.5 bg-red-400 text-white rounded-lg">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="w-6 h-6"
>
<path
d="M7.5 3.375c0-1.036.84-1.875 1.875-1.875h.375a3.75 3.75 0 0 1 3.75 3.75v1.875C13.5 8.161 14.34 9 15.375 9h1.875A3.75 3.75 0 0 1 21 12.75v3.375C21 17.16 20.16 18 19.125 18h-9.75A1.875 1.875 0 0 1 7.5 16.125V3.375Z"
/>
<path
d="M15 5.25a5.23 5.23 0 0 0-1.279-3.434 9.768 9.768 0 0 1 6.963 6.963A5.23 5.23 0 0 0 17.25 7.5h-1.875A.375.375 0 0 1 15 7.125V5.25ZM4.875 6H6v10.125A3.375 3.375 0 0 0 9.375 19.5H16.5v1.125c0 1.035-.84 1.875-1.875 1.875h-9.75A1.875 1.875 0 0 1 3 20.625V7.875C3 6.839 3.84 6 4.875 6Z"
/>
</svg>
</div>
<div class="flex flex-col justify-center -space-y-0.5">
<div class=" dark:text-gray-100 text-sm font-medium line-clamp-1">
{file?.title ?? `#${file.name}`}
</div>
<div class=" text-gray-500 text-sm">Collection</div>
</div>
</div>
{/if}
<div class=" absolute -top-1 -right-1">
......
src/lib/components/chat/MessageInput/Documents.svelte
View file @ 1f02940b
......@@ -10,14 +10,50 @@
const dispatch = createEventDispatcher();
let selectedIdx = 0;
let filteredItems = [];
let filteredDocs = [];
let collections = [];
$: collections = [
...($documents.length > 0
? [
{
name: 'All Documents',
type: 'collection',
title: 'All Documents',
collection_names: $documents.map((doc) => doc.collection_name)
}
]
: []),
...$documents
.reduce((a, e, i, arr) => {
return [...new Set([...a, ...(e?.content?.tags ?? []).map((tag) => tag.name)])];
}, [])
.map((tag) => ({
name: tag,
type: 'collection',
collection_names: $documents
.filter((doc) => (doc?.content?.tags ?? []).map((tag) => tag.name).includes(tag))
.map((doc) => doc.collection_name)
}))
];
$: filteredCollections = collections
.filter((collection) => collection.name.includes(prompt.split(' ')?.at(0)?.substring(1) ?? ''))
.sort((a, b) => a.name.localeCompare(b.name));
$: filteredDocs = $documents
.filter((p) => p.name.includes(prompt.split(' ')?.at(0)?.substring(1) ?? ''))
.filter((doc) => doc.name.includes(prompt.split(' ')?.at(0)?.substring(1) ?? ''))
.sort((a, b) => a.title.localeCompare(b.title));
$: filteredItems = [...filteredCollections, ...filteredDocs];
$: if (prompt) {
selectedIdx = 0;
console.log(filteredCollections);
}
export const selectUp = () => {
......@@ -25,7 +61,7 @@
};
export const selectDown = () => {
selectedIdx = Math.min(selectedIdx + 1, filteredDocs.length - 1);
selectedIdx = Math.min(selectedIdx + 1, filteredItems.length - 1);
};
const confirmSelect = async (doc) => {
......@@ -51,7 +87,7 @@
};
</script>
{#if filteredDocs.length > 0 || prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
{#if filteredItems.length > 0 || prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
<div class="md:px-2 mb-3 text-left w-full">
<div class="flex w-full rounded-lg border border-gray-100 dark:border-gray-700">
<div class=" bg-gray-100 dark:bg-gray-700 w-10 rounded-l-lg text-center">
......@@ -60,7 +96,7 @@
<div class="max-h-60 flex flex-col w-full rounded-r-lg">
<div class=" overflow-y-auto bg-white p-2 rounded-tr-lg space-y-0.5">
{#each filteredDocs as doc, docIdx}
{#each filteredItems as doc, docIdx}
<button
class=" px-3 py-1.5 rounded-lg w-full text-left {docIdx === selectedIdx
? ' bg-gray-100 selected-command-option-button'
......@@ -68,6 +104,7 @@
type="button"
on:click={() => {
console.log(doc);
confirmSelect(doc);
}}
on:mousemove={() => {
......@@ -75,13 +112,21 @@
}}
on:focus={() => {}}
>
<div class=" font-medium text-black line-clamp-1">
#{doc.name} ({doc.filename})
</div>
<div class=" text-xs text-gray-600 line-clamp-1">
{doc.title}
</div>
{#if doc.type === 'collection'}
<div class=" font-medium text-black line-clamp-1">
{doc?.title ?? `#${doc.name}`}
</div>
<div class=" text-xs text-gray-600 line-clamp-1">Collection</div>
{:else}
<div class=" font-medium text-black line-clamp-1">
#{doc.name} ({doc.filename})
</div>
<div class=" text-xs text-gray-600 line-clamp-1">
{doc.title}
</div>
{/if}
</button>
{/each}
......
src/lib/components/chat/Messages/UserMessage.svelte
View file @ 1f02940b
......@@ -117,6 +117,35 @@
<div class=" text-gray-500 text-sm">Document</div>
</div>
</button>
{:else if file.type === 'collection'}
<button
class="h-16 w-[15rem] flex items-center space-x-3 px-2.5 dark:bg-gray-600 rounded-xl border border-gray-200 dark:border-none text-left"
type="button"
>
<div class="p-2.5 bg-red-400 text-white rounded-lg">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="w-6 h-6"
>
<path
d="M7.5 3.375c0-1.036.84-1.875 1.875-1.875h.375a3.75 3.75 0 0 1 3.75 3.75v1.875C13.5 8.161 14.34 9 15.375 9h1.875A3.75 3.75 0 0 1 21 12.75v3.375C21 17.16 20.16 18 19.125 18h-9.75A1.875 1.875 0 0 1 7.5 16.125V3.375Z"
/>
<path
d="M15 5.25a5.23 5.23 0 0 0-1.279-3.434 9.768 9.768 0 0 1 6.963 6.963A5.23 5.23 0 0 0 17.25 7.5h-1.875A.375.375 0 0 1 15 7.125V5.25ZM4.875 6H6v10.125A3.375 3.375 0 0 0 9.375 19.5H16.5v1.125c0 1.035-.84 1.875-1.875 1.875h-9.75A1.875 1.875 0 0 1 3 20.625V7.875C3 6.839 3.84 6 4.875 6Z"
/>
</svg>
</div>
<div class="flex flex-col justify-center -space-y-0.5">
<div class=" dark:text-gray-100 text-sm font-medium line-clamp-1">
{file?.title ?? `#${file.name}`}
</div>
<div class=" text-gray-500 text-sm">Collection</div>
</div>
</button>
{/if}
</div>
{/each}
......
src/lib/components/common/Tags.svelte 0 → 100644
View file @ 1f02940b
<script lang="ts">
import TagInput from './Tags/TagInput.svelte';
import TagList from './Tags/TagList.svelte';
export let tags = [];
export let deleteTag: Function;
export let addTag: Function;
</script>
<div class="flex flex-row space-x-0.5 line-clamp-1">
<TagList
{tags}
on:delete={(e) => {
deleteTag(e.detail);
}}
/>
<TagInput
on:add={(e) => {
addTag(e.detail);
}}
/>
</div>
src/lib/components/common/Tags/TagInput.svelte 0 → 100644
View file @ 1f02940b
<script lang="ts">
import { createEventDispatcher } from 'svelte';
const dispatch = createEventDispatcher();
let showTagInput = false;
let tagName = '';
</script>
<div class="flex space-x-1 pl-1.5">
{#if showTagInput}
<div class="flex items-center">
<input
bind:value={tagName}
class=" cursor-pointer self-center text-xs h-fit bg-transparent outline-none line-clamp-1 w-[4rem]"
placeholder="Add a tag"
/>
<button
type="button"
on:click={() => {
dispatch('add', tagName);
tagName = '';
showTagInput = false;
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-3 h-3"
>
<path
fill-rule="evenodd"
d="M12.416 3.376a.75.75 0 0 1 .208 1.04l-5 7.5a.75.75 0 0 1-1.154.114l-3-3a.75.75 0 0 1 1.06-1.06l2.353 2.353 4.493-6.74a.75.75 0 0 1 1.04-.207Z"
clip-rule="evenodd"
/>
</svg>
</button>
</div>
<!-- TODO: Tag Suggestions -->
{/if}
<button
class=" cursor-pointer self-center p-0.5 space-x-1 flex h-fit items-center dark:hover:bg-gray-700 rounded-full transition border dark:border-gray-600 border-dashed"
type="button"
on:click={() => {
showTagInput = !showTagInput;
}}
>
<div class=" m-auto self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-3 h-3 {showTagInput ? 'rotate-45' : ''} transition-all transform"
>
<path
d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
/>
</svg>
</div>
</button>
</div>
src/lib/components/common/Tags/TagList.svelte 0 → 100644
View file @ 1f02940b
<script lang="ts">
import { createEventDispatcher } from 'svelte';
const dispatch = createEventDispatcher();
export let tags = [];
</script>
{#each tags as tag}
<div
class="px-2 py-0.5 space-x-1 flex h-fit items-center rounded-full transition border dark:border-gray-600 dark:text-white"
>
<div class=" text-[0.7rem] font-medium self-center line-clamp-1">
{tag.name}
</div>
<button
class=" m-auto self-center cursor-pointer"
on:click={() => {
dispatch('delete', tag.name);
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-3 h-3"
>
<path
d="M5.28 4.22a.75.75 0 0 0-1.06 1.06L6.94 8l-2.72 2.72a.75.75 0 1 0 1.06 1.06L8 9.06l2.72 2.72a.75.75 0 1 0 1.06-1.06L9.06 8l2.72-2.72a.75.75 0 0 0-1.06-1.06L8 6.94 5.28 4.22Z"
/>
</svg>
</button>
</div>
{/each}
src/lib/components/documents/EditDocModal.svelte
View file @ 1f02940b
......@@ -3,16 +3,22 @@
import dayjs from 'dayjs';
import { onMount } from 'svelte';
import { getDocs, updateDocByName } from '$lib/apis/documents';
import { getDocs, tagDocByName, updateDocByName } from '$lib/apis/documents';
import Modal from '../common/Modal.svelte';
import { documents } from '$lib/stores';
import TagInput from '../common/Tags/TagInput.svelte';
import Tags from '../common/Tags.svelte';
import { addTagById } from '$lib/apis/chats';
export let show = false;
export let selectedDoc;
let tags = [];
let doc = {
name: '',
title: ''
title: '',
content: null
};
const submitHandler = async () => {
......@@ -30,9 +36,37 @@
}
};
const addTagHandler = async (tagName) => {
if (!tags.find((tag) => tag.name === tagName) && tagName !== '') {
tags = [...tags, { name: tagName }];
await tagDocByName(localStorage.token, doc.name, {
name: doc.name,
tags: tags
});
documents.set(await getDocs(localStorage.token));
} else {
console.log('tag already exists');
}
};
const deleteTagHandler = async (tagName) => {
tags = tags.filter((tag) => tag.name !== tagName);
await tagDocByName(localStorage.token, doc.name, {
name: doc.name,
tags: tags
});
documents.set(await getDocs(localStorage.token));
};
onMount(() => {
if (selectedDoc) {
doc = JSON.parse(JSON.stringify(selectedDoc));
tags = doc?.content?.tags ?? [];
}
});
</script>
......@@ -112,6 +146,12 @@
/>
</div>
</div>
<div class="flex flex-col w-full">
<div class=" mb-1.5 text-xs text-gray-500">Tags</div>
<Tags {tags} addTag={addTagHandler} deleteTag={deleteTagHandler} />
</div>
</div>
<div class="flex justify-end pt-5 text-sm font-medium">
......
src/lib/components/layout/Navbar.svelte
View file @ 1f02940b
......@@ -6,6 +6,8 @@
import { getChatById } from '$lib/apis/chats';
import { chatId, modelfiles } from '$lib/stores';
import ShareChatModal from '../chat/ShareChatModal.svelte';
import TagInput from '../common/Tags/TagInput.svelte';
import Tags from '../common/Tags.svelte';
export let initNewChat: Function;
export let title: string = 'Ollama Web UI';
......@@ -61,21 +63,6 @@
saveAs(blob, `chat-${chat.title}.txt`);
};
const addTagHandler = () => {
// if (!tags.find((e) => e.name === tagName)) {
// tags = [
// ...tags,
// {
// name: JSON.parse(JSON.stringify(tagName))
// }
// ];
// }
addTag(tagName);
tagName = '';
showTagInput = false;
};
</script>
<ShareChatModal bind:show={showShareChatModal} {downloadChat} {shareChat} />
......@@ -116,87 +103,7 @@
<div class="pl-2 self-center flex items-center space-x-2">
{#if shareEnabled}
<div class="flex flex-row space-x-0.5 line-clamp-1">
{#each tags as tag}
<div
class="px-2 py-0.5 space-x-1 flex h-fit items-center rounded-full transition border dark:border-gray-600 dark:text-white"
>
<div class=" text-[0.65rem] font-medium self-center line-clamp-1">
{tag.name}
</div>
<button
class=" m-auto self-center cursor-pointer"
on:click={() => {
deleteTag(tag.name);
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-3 h-3"
>
<path
d="M5.28 4.22a.75.75 0 0 0-1.06 1.06L6.94 8l-2.72 2.72a.75.75 0 1 0 1.06 1.06L8 9.06l2.72 2.72a.75.75 0 1 0 1.06-1.06L9.06 8l2.72-2.72a.75.75 0 0 0-1.06-1.06L8 6.94 5.28 4.22Z"
/>
</svg>
</button>
</div>
{/each}
<div class="flex space-x-1 pl-1.5">
{#if showTagInput}
<div class="flex items-center">
<input
bind:value={tagName}
class=" cursor-pointer self-center text-xs h-fit bg-transparent outline-none line-clamp-1 w-[4rem]"
placeholder="Add a tag"
/>
<button
on:click={() => {
addTagHandler();
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-3 h-3"
>
<path
fill-rule="evenodd"
d="M12.416 3.376a.75.75 0 0 1 .208 1.04l-5 7.5a.75.75 0 0 1-1.154.114l-3-3a.75.75 0 0 1 1.06-1.06l2.353 2.353 4.493-6.74a.75.75 0 0 1 1.04-.207Z"
clip-rule="evenodd"
/>
</svg>
</button>
</div>
<!-- TODO: Tag Suggestions -->
{/if}
<button
class=" cursor-pointer self-center p-0.5 space-x-1 flex h-fit items-center dark:hover:bg-gray-700 rounded-full transition border dark:border-gray-600 border-dashed"
on:click={() => {
showTagInput = !showTagInput;
}}
>
<div class=" m-auto self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-3 h-3 {showTagInput ? 'rotate-45' : ''} transition-all transform"
>
<path
d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
/>
</svg>
</div>
</button>
</div>
</div>
<Tags {tags} {deleteTag} {addTag} />
<button
class=" cursor-pointer p-1.5 flex dark:hover:bg-gray-700 rounded-lg transition border dark:border-gray-600"
......
src/routes/(app)/+page.svelte
View file @ 1f02940b
......@@ -28,7 +28,7 @@
getTagsById,
updateChatById
} from '$lib/apis/chats';
import { queryVectorDB } from '$lib/apis/rag';
import { queryCollection, queryDoc } from '$lib/apis/rag';
import { generateOpenAIChatCompletion } from '$lib/apis/openai';
import MessageInput from '$lib/components/chat/MessageInput.svelte';
......@@ -224,7 +224,9 @@
const docs = messages
.filter((message) => message?.files ?? null)
.map((message) => message.files.filter((item) => item.type === 'doc'))
.map((message) =>
message.files.filter((item) => item.type === 'doc' || item.type === 'collection')
)
.flat(1);
console.log(docs);
......@@ -234,12 +236,21 @@
let relevantContexts = await Promise.all(
docs.map(async (doc) => {
return await queryVectorDB(localStorage.token, doc.collection_name, query, 4).catch(
(error) => {
console.log(error);
return null;
}
);
if (doc.type === 'collection') {
return await queryCollection(localStorage.token, doc.collection_names, query, 4).catch(
(error) => {
console.log(error);
return null;
}
);
} else {
return await queryDoc(localStorage.token, doc.collection_name, query, 4).catch(
(error) => {
console.log(error);
return null;
}
);
}
})
);
relevantContexts = relevantContexts.filter((context) => context);
......
src/routes/(app)/c/[id]/+page.svelte
View file @ 1f02940b
......@@ -29,7 +29,7 @@
getTagsById,
updateChatById
} from '$lib/apis/chats';
import { queryVectorDB } from '$lib/apis/rag';
import { queryCollection, queryDoc } from '$lib/apis/rag';
import { generateOpenAIChatCompletion } from '$lib/apis/openai';
import MessageInput from '$lib/components/chat/MessageInput.svelte';
......@@ -238,7 +238,9 @@
const docs = messages
.filter((message) => message?.files ?? null)
.map((message) => message.files.filter((item) => item.type === 'doc'))
.map((message) =>
message.files.filter((item) => item.type === 'doc' || item.type === 'collection')
)
.flat(1);
console.log(docs);
......@@ -248,12 +250,21 @@
let relevantContexts = await Promise.all(
docs.map(async (doc) => {
return await queryVectorDB(localStorage.token, doc.collection_name, query, 4).catch(
(error) => {
console.log(error);
return null;
}
);
if (doc.type === 'collection') {
return await queryCollection(localStorage.token, doc.collection_names, query, 4).catch(
(error) => {
console.log(error);
return null;
}
);
} else {
return await queryDoc(localStorage.token, doc.collection_name, query, 4).catch(
(error) => {
console.log(error);
return null;
}
);
}
})
);
relevantContexts = relevantContexts.filter((context) => context);
......
src/routes/(app)/documents/+page.svelte
View file @ 1f02940b
......@@ -12,14 +12,17 @@
import { transformFileName } from '$lib/utils';
import EditDocModal from '$lib/components/documents/EditDocModal.svelte';
import AddFilesPlaceholder from '$lib/components/AddFilesPlaceholder.svelte';
let importFiles = '';
let inputFiles = '';
let query = '';
let tags = [];
let showEditDocModal = false;
let selectedDoc;
let selectedTag = '';
let dragged = false;
......@@ -49,48 +52,131 @@
}
};
const onDragOver = (e) => {
e.preventDefault();
dragged = true;
};
const onDragLeave = () => {
dragged = false;
};
const onDrop = async (e) => {
e.preventDefault();
console.log(e);
if (e.dataTransfer?.files) {
const inputFiles = e.dataTransfer?.files;
if (inputFiles && inputFiles.length > 0) {
const file = inputFiles[0];
if (
SUPPORTED_FILE_TYPE.includes(file['type']) ||
SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1))
) {
uploadDoc(file);
onMount(() => {
documents.subscribe((docs) => {
tags = docs.reduce((a, e, i, arr) => {
return [...new Set([...a, ...(e?.content?.tags ?? []).map((tag) => tag.name)])];
}, []);
});
const dropZone = document.querySelector('body');
const onDragOver = (e) => {
e.preventDefault();
dragged = true;
};
const onDragLeave = () => {
dragged = false;
};
const onDrop = async (e) => {
e.preventDefault();
console.log(e);
if (e.dataTransfer?.files) {
let reader = new FileReader();
reader.onload = (event) => {
files = [
...files,
{
type: 'image',
url: `${event.target.result}`
}
];
};
const inputFiles = e.dataTransfer?.files;
if (inputFiles && inputFiles.length > 0) {
const file = inputFiles[0];
console.log(file, file.name.split('.').at(-1));
if (['image/gif', 'image/jpeg', 'image/png'].includes(file['type'])) {
reader.readAsDataURL(file);
} else if (
SUPPORTED_FILE_TYPE.includes(file['type']) ||
SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1))
) {
uploadDoc(file);
} else {
toast.error(
`Unknown File Type '${file['type']}', but accepting and treating as plain text`
);
uploadDoc(file);
}
} else {
toast.error(
`Unknown File Type '${file['type']}', but accepting and treating as plain text`
);
uploadDoc(file);
toast.error(`File not found.`);
}
} else {
toast.error(`File not found.`);
}
}
dragged = false;
};
dragged = false;
};
dropZone?.addEventListener('dragover', onDragOver);
dropZone?.addEventListener('drop', onDrop);
dropZone?.addEventListener('dragleave', onDragLeave);
return () => {
dropZone?.removeEventListener('dragover', onDragOver);
dropZone?.removeEventListener('drop', onDrop);
dropZone?.removeEventListener('dragleave', onDragLeave);
};
});
</script>
{#if dragged}
<div
class="fixed w-full h-full flex z-50 touch-none pointer-events-none"
id="dropzone"
role="region"
aria-label="Drag and Drop Container"
>
<div class="absolute rounded-xl w-full h-full backdrop-blur bg-gray-800/40 flex justify-center">
<div class="m-auto pt-64 flex flex-col justify-center">
<div class="max-w-md">
<AddFilesPlaceholder>
<div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
Drop any files here to add to my documents
</div>
</AddFilesPlaceholder>
</div>
</div>
</div>
</div>
{/if}
{#key selectedDoc}
<EditDocModal bind:show={showEditDocModal} {selectedDoc} />
{/key}
<input
id="upload-doc-input"
bind:files={inputFiles}
type="file"
hidden
on:change={async (e) => {
if (inputFiles && inputFiles.length > 0) {
const file = inputFiles[0];
if (
SUPPORTED_FILE_TYPE.includes(file['type']) ||
SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1))
) {
uploadDoc(file);
} else {
toast.error(
`Unknown File Type '${file['type']}', but accepting and treating as plain text`
);
uploadDoc(file);
}
inputFiles = null;
e.target.value = '';
} else {
toast.error(`File not found.`);
}
}}
/>
<div class="min-h-screen w-full flex justify-center dark:text-white">
<div class=" py-2.5 flex flex-col justify-between w-full">
<div class="max-w-2xl mx-auto w-full px-3 md:px-0 my-10">
......@@ -141,36 +227,36 @@
</button>
</div>
</div>
<hr class=" dark:border-gray-700 my-2.5" />
<input
id="upload-doc-input"
bind:files={inputFiles}
type="file"
hidden
on:change={async (e) => {
if (inputFiles && inputFiles.length > 0) {
const file = inputFiles[0];
if (
SUPPORTED_FILE_TYPE.includes(file['type']) ||
SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1))
) {
uploadDoc(file);
} else {
toast.error(
`Unknown File Type '${file['type']}', but accepting and treating as plain text`
);
uploadDoc(file);
}
inputFiles = null;
e.target.value = '';
} else {
toast.error(`File not found.`);
}
}}
/>
{#if tags.length > 0}
<div class="px-2.5 mt-0.5 mb-2 flex gap-1 flex-wrap">
<button
class="px-2 py-0.5 space-x-1 flex h-fit items-center rounded-full transition border dark:border-gray-600 dark:text-white"
on:click={async () => {
selectedTag = '';
// await chats.set(await getChatListByTagName(localStorage.token, tag.name));
}}
>
<div class=" text-xs font-medium self-center line-clamp-1">all</div>
</button>
{#each tags as tag}
<button
class="px-2 py-0.5 space-x-1 flex h-fit items-center rounded-full transition border dark:border-gray-600 dark:text-white"
on:click={async () => {
selectedTag = tag;
// await chats.set(await getChatListByTagName(localStorage.token, tag.name));
}}
>
<div class=" text-xs font-medium self-center line-clamp-1">
#{tag}
</div>
</button>
{/each}
</div>
{/if}
<div>
<!-- <div>
<div
class="my-3 py-16 rounded-lg border-2 border-dashed dark:border-gray-600 {dragged &&
' dark:bg-gray-700'} "
......@@ -187,11 +273,12 @@
</div>
</div>
</div>
</div>
</div> -->
{#each $documents.filter((p) => query === '' || p.name.includes(query)) as doc}
<hr class=" dark:border-gray-700 my-2.5" />
<div class=" flex space-x-4 cursor-pointer w-full mb-3">
{#each $documents.filter((doc) => (selectedTag === '' || (doc?.content?.tags ?? [])
.map((tag) => tag.name)
.includes(selectedTag)) && (query === '' || doc.name.includes(query))) as doc}
<div class=" flex space-x-4 cursor-pointer w-full mt-3 mb-3">
<div class=" flex flex-1 space-x-4 cursor-pointer w-full">
<div class=" flex items-center space-x-3">
<div class="p-2.5 bg-red-400 text-white rounded-lg">
......@@ -330,106 +417,97 @@
</div>
</div>
{/each}
{#if $documents.length != 0}
<hr class=" dark:border-gray-700 my-2.5" />
<div class=" flex justify-between w-full mb-3">
<div class="flex space-x-2">
<input
id="documents-import-input"
bind:files={importFiles}
type="file"
accept=".json"
hidden
on:change={() => {
console.log(importFiles);
const reader = new FileReader();
reader.onload = async (event) => {
const savedDocs = JSON.parse(event.target.result);
console.log(savedDocs);
for (const doc of savedDocs) {
await createNewDoc(
localStorage.token,
doc.collection_name,
doc.filename,
doc.name,
doc.title
).catch((error) => {
toast.error(error);
return null;
});
}
await documents.set(await getDocs(localStorage.token));
};
reader.readAsText(importFiles[0]);
}}
/>
<button
class="self-center w-fit text-sm px-3 py-1 border dark:border-gray-600 rounded-xl flex"
on:click={async () => {
document.getElementById('documents-import-input')?.click();
}}
>
<div class=" self-center mr-2 font-medium">Import Documents Mapping</div>
<div class=" self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M4 2a1.5 1.5 0 0 0-1.5 1.5v9A1.5 1.5 0 0 0 4 14h8a1.5 1.5 0 0 0 1.5-1.5V6.621a1.5 1.5 0 0 0-.44-1.06L9.94 2.439A1.5 1.5 0 0 0 8.878 2H4Zm4 9.5a.75.75 0 0 1-.75-.75V8.06l-.72.72a.75.75 0 0 1-1.06-1.06l2-2a.75.75 0 0 1 1.06 0l2 2a.75.75 0 1 1-1.06 1.06l-.72-.72v2.69a.75.75 0 0 1-.75.75Z"
clip-rule="evenodd"
/>
</svg>
</div>
</button>
<hr class=" dark:border-gray-700 my-2.5" />
<button
class="self-center w-fit text-sm px-3 py-1 border dark:border-gray-600 rounded-xl flex"
on:click={async () => {
let blob = new Blob([JSON.stringify($documents)], {
type: 'application/json'
});
saveAs(blob, `documents-mapping-export-${Date.now()}.json`);
}}
>
<div class=" self-center mr-2 font-medium">Export Documents Mapping</div>
<div class=" self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M4 2a1.5 1.5 0 0 0-1.5 1.5v9A1.5 1.5 0 0 0 4 14h8a1.5 1.5 0 0 0 1.5-1.5V6.621a1.5 1.5 0 0 0-.44-1.06L9.94 2.439A1.5 1.5 0 0 0 8.878 2H4Zm4 3.5a.75.75 0 0 1 .75.75v2.69l.72-.72a.75.75 0 1 1 1.06 1.06l-2 2a.75.75 0 0 1-1.06 0l-2-2a.75.75 0 0 1 1.06-1.06l.72.72V6.25A.75.75 0 0 1 8 5.5Z"
clip-rule="evenodd"
/>
</svg>
</div>
</button>
<div class=" flex justify-between w-full mb-3">
<div class="flex space-x-2">
<input
id="documents-import-input"
bind:files={importFiles}
type="file"
accept=".json"
hidden
on:change={() => {
console.log(importFiles);
const reader = new FileReader();
reader.onload = async (event) => {
const savedDocs = JSON.parse(event.target.result);
console.log(savedDocs);
for (const doc of savedDocs) {
await createNewDoc(
localStorage.token,
doc.collection_name,
doc.filename,
doc.name,
doc.title
).catch((error) => {
toast.error(error);
return null;
});
}
await documents.set(await getDocs(localStorage.token));
};
reader.readAsText(importFiles[0]);
}}
/>
<!-- <button
on:click={() => {
loadDefaultPrompts();
<button
class="self-center w-fit text-sm px-3 py-1 border dark:border-gray-600 rounded-xl flex"
on:click={async () => {
document.getElementById('documents-import-input')?.click();
}}
>
dd
</button> -->
</div>
<div class=" self-center mr-2 font-medium">Import Documents Mapping</div>
<div class=" self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M4 2a1.5 1.5 0 0 0-1.5 1.5v9A1.5 1.5 0 0 0 4 14h8a1.5 1.5 0 0 0 1.5-1.5V6.621a1.5 1.5 0 0 0-.44-1.06L9.94 2.439A1.5 1.5 0 0 0 8.878 2H4Zm4 9.5a.75.75 0 0 1-.75-.75V8.06l-.72.72a.75.75 0 0 1-1.06-1.06l2-2a.75.75 0 0 1 1.06 0l2 2a.75.75 0 1 1-1.06 1.06l-.72-.72v2.69a.75.75 0 0 1-.75.75Z"
clip-rule="evenodd"
/>
</svg>
</div>
</button>
<button
class="self-center w-fit text-sm px-3 py-1 border dark:border-gray-600 rounded-xl flex"
on:click={async () => {
let blob = new Blob([JSON.stringify($documents)], {
type: 'application/json'
});
saveAs(blob, `documents-mapping-export-${Date.now()}.json`);
}}
>
<div class=" self-center mr-2 font-medium">Export Documents Mapping</div>
<div class=" self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M4 2a1.5 1.5 0 0 0-1.5 1.5v9A1.5 1.5 0 0 0 4 14h8a1.5 1.5 0 0 0 1.5-1.5V6.621a1.5 1.5 0 0 0-.44-1.06L9.94 2.439A1.5 1.5 0 0 0 8.878 2H4Zm4 3.5a.75.75 0 0 1 .75.75v2.69l.72-.72a.75.75 0 1 1 1.06 1.06l-2 2a.75.75 0 0 1-1.06 0l-2-2a.75.75 0 0 1 1.06-1.06l.72.72V6.25A.75.75 0 0 1 8 5.5Z"
clip-rule="evenodd"
/>
</svg>
</div>
</button>
</div>
{/if}
</div>
<div class="text-xs flex items-center space-x-1">
<div>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment