Skip to content

GitLab

Unverified Commit f6a5d4b0 authored by Timothy Jaeryang Baek's avatar Timothy Jaeryang Baek Committed by GitHub
Browse files

Merge pull request #424 from ollama-webui/documents 

feat: full documents support
parents 34e54454 edceeba1
Show whitespace changes
Inline Side-by-side
Showing with 1283 additions and 55 deletions
backend/apps/rag/main.py
View file @ f6a5d4b0
......@@ -119,7 +119,11 @@ def store_web(form_data: StoreWebForm, user=Depends(get_current_user)):
loader = WebBaseLoader(form_data.url)
data = loader.load()
store_data_in_vector_db(data, form_data.collection_name)
return {"status": True, "collection_name": form_data.collection_name}
return {
"status": True,
"collection_name": form_data.collection_name,
"filename": form_data.url,
}
except Exception as e:
print(e)
raise HTTPException(
......@@ -176,7 +180,11 @@ def store_doc(
result = store_data_in_vector_db(data, collection_name)
if result:
return {"status": True, "collection_name": collection_name}
return {
"status": True,
"collection_name": collection_name,
"filename": filename,
}
else:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
......
backend/apps/web/main.py
View file @ f6a5d4b0
from fastapi import FastAPI, Depends
from fastapi.routing import APIRoute
from fastapi.middleware.cors import CORSMiddleware
from apps.web.routers import auths, users, chats, modelfiles, prompts, configs, utils
from apps.web.routers import (
auths,
users,
chats,
documents,
modelfiles,
prompts,
configs,
utils,
)
from config import WEBUI_VERSION, WEBUI_AUTH
app = FastAPI()
......@@ -22,9 +31,8 @@ app.add_middleware(
app.include_router(auths.router, prefix="/auths", tags=["auths"])
app.include_router(users.router, prefix="/users", tags=["users"])
app.include_router(chats.router, prefix="/chats", tags=["chats"])
app.include_router(modelfiles.router,
prefix="/modelfiles",
tags=["modelfiles"])
app.include_router(documents.router, prefix="/documents", tags=["documents"])
app.include_router(modelfiles.router, prefix="/modelfiles", tags=["modelfiles"])
app.include_router(prompts.router, prefix="/prompts", tags=["prompts"])
app.include_router(configs.router, prefix="/configs", tags=["configs"])
......
backend/apps/web/models/documents.py 0 → 100644
View file @ f6a5d4b0
from pydantic import BaseModel
from peewee import *
from playhouse.shortcuts import model_to_dict
from typing import List, Union, Optional
import time
from utils.utils import decode_token
from utils.misc import get_gravatar_url
from apps.web.internal.db import DB
import json
####################
# Documents DB Schema
####################
class Document(Model):
collection_name = CharField(unique=True)
name = CharField(unique=True)
title = CharField()
filename = CharField()
content = TextField(null=True)
user_id = CharField()
timestamp = DateField()
class Meta:
database = DB
class DocumentModel(BaseModel):
collection_name: str
name: str
title: str
filename: str
content: Optional[str] = None
user_id: str
timestamp: int # timestamp in epoch
####################
# Forms
####################
class DocumentUpdateForm(BaseModel):
name: str
title: str
class DocumentForm(DocumentUpdateForm):
collection_name: str
filename: str
content: Optional[str] = None
class DocumentsTable:
def __init__(self, db):
self.db = db
self.db.create_tables([Document])
def insert_new_doc(
self, user_id: str, form_data: DocumentForm
) -> Optional[DocumentModel]:
document = DocumentModel(
**{
**form_data.model_dump(),
"user_id": user_id,
"timestamp": int(time.time()),
}
)
try:
result = Document.create(**document.model_dump())
if result:
return document
else:
return None
except:
return None
def get_doc_by_name(self, name: str) -> Optional[DocumentModel]:
try:
document = Document.get(Document.name == name)
return DocumentModel(**model_to_dict(document))
except:
return None
def get_docs(self) -> List[DocumentModel]:
return [
DocumentModel(**model_to_dict(doc))
for doc in Document.select()
# .limit(limit).offset(skip)
]
def update_doc_by_name(
self, name: str, form_data: DocumentUpdateForm
) -> Optional[DocumentModel]:
try:
query = Document.update(
title=form_data.title,
name=form_data.name,
timestamp=int(time.time()),
).where(Document.name == name)
query.execute()
doc = Document.get(Document.name == form_data.name)
return DocumentModel(**model_to_dict(doc))
except Exception as e:
print(e)
return None
def delete_doc_by_name(self, name: str) -> bool:
try:
query = Document.delete().where((Document.name == name))
query.execute() # Remove the rows, return number of rows removed.
return True
except:
return False
Documents = DocumentsTable(DB)
backend/apps/web/routers/documents.py 0 → 100644
View file @ f6a5d4b0
from fastapi import Depends, FastAPI, HTTPException, status
from datetime import datetime, timedelta
from typing import List, Union, Optional
from fastapi import APIRouter
from pydantic import BaseModel
import json
from apps.web.models.documents import (
Documents,
DocumentForm,
DocumentUpdateForm,
DocumentModel,
)
from utils.utils import get_current_user
from constants import ERROR_MESSAGES
router = APIRouter()
############################
# GetDocuments
############################
@router.get("/", response_model=List[DocumentModel])
async def get_documents(user=Depends(get_current_user)):
return Documents.get_docs()
############################
# CreateNewDoc
############################
@router.post("/create", response_model=Optional[DocumentModel])
async def create_new_doc(form_data: DocumentForm, user=Depends(get_current_user)):
if user.role != "admin":
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
)
doc = Documents.get_doc_by_name(form_data.name)
if doc == None:
doc = Documents.insert_new_doc(user.id, form_data)
if doc:
return doc
else:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.FILE_EXISTS,
)
else:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NAME_TAG_TAKEN,
)
############################
# GetDocByName
############################
@router.get("/name/{name}", response_model=Optional[DocumentModel])
async def get_doc_by_name(name: str, user=Depends(get_current_user)):
doc = Documents.get_doc_by_name(name)
if doc:
return doc
else:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=ERROR_MESSAGES.NOT_FOUND,
)
############################
# UpdateDocByName
############################
@router.post("/name/{name}/update", response_model=Optional[DocumentModel])
async def update_doc_by_name(
name: str, form_data: DocumentUpdateForm, user=Depends(get_current_user)
):
if user.role != "admin":
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
)
doc = Documents.update_doc_by_name(name, form_data)
if doc:
return doc
else:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NAME_TAG_TAKEN,
)
############################
# DeleteDocByName
############################
@router.delete("/name/{name}/delete", response_model=bool)
async def delete_doc_by_name(name: str, user=Depends(get_current_user)):
if user.role != "admin":
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
)
result = Documents.delete_doc_by_name(name)
return result
backend/config.py
View file @ f6a5d4b0
......@@ -58,7 +58,7 @@ if OPENAI_API_BASE_URL == "":
# WEBUI_VERSION
####################################
WEBUI_VERSION = os.environ.get("WEBUI_VERSION", "v1.0.0-alpha.50")
WEBUI_VERSION = os.environ.get("WEBUI_VERSION", "v1.0.0-alpha.61")
####################################
# WEBUI_AUTH (Required for security)
......
backend/constants.py
View file @ f6a5d4b0
......@@ -18,6 +18,9 @@ class ERROR_MESSAGES(str, Enum):
"Uh-oh! This username is already registered. Please choose another username."
)
COMMAND_TAKEN = "Uh-oh! This command is already registered. Please choose another command string."
FILE_EXISTS = "Uh-oh! This file is already registered. Please choose another file."
NAME_TAG_TAKEN = "Uh-oh! This name tag is already registered. Please choose another name tag string."
INVALID_TOKEN = (
"Your session has expired or the token is invalid. Please sign in again."
)
......
src/lib/apis/documents/index.ts 0 → 100644
View file @ f6a5d4b0
import { WEBUI_API_BASE_URL } from '$lib/constants';
export const createNewDoc = async (
token: string,
collection_name: string,
filename: string,
name: string,
title: string
) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/documents/create`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify({
collection_name: collection_name,
filename: filename,
name: name,
title: title
})
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const getDocs = async (token: string = '') => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/documents/`, {
method: 'GET',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.then((json) => {
return json;
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const getDocByName = async (token: string, name: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/documents/name/${name}`, {
method: 'GET',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.then((json) => {
return json;
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};
type DocUpdateForm = {
name: string;
title: string;
};
export const updateDocByName = async (token: string, name: string, form: DocUpdateForm) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/documents/name/${name}/update`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify({
name: form.name,
title: form.title
})
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.then((json) => {
return json;
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const deleteDocByName = async (token: string, name: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/documents/name/${name}/delete`, {
method: 'DELETE',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.then((json) => {
return json;
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};
src/lib/components/AddFilesPlaceholder.svelte 0 → 100644
View file @ f6a5d4b0
<div class=" text-center text-6xl mb-3">📄</div>
<div class="text-center dark:text-white text-2xl font-semibold z-50">Add Files</div>
<div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
Drop any files here to add to the conversation
</div>
src/lib/components/chat/MessageInput.svelte
View file @ f6a5d4b0
......@@ -7,6 +7,9 @@
import Prompts from './MessageInput/PromptCommands.svelte';
import Suggestions from './MessageInput/Suggestions.svelte';
import { uploadDocToVectorDB } from '$lib/apis/rag';
import AddFilesPlaceholder from '../AddFilesPlaceholder.svelte';
import { SUPPORTED_FILE_TYPE } from '$lib/constants';
import Documents from './MessageInput/Documents.svelte';
export let submitPrompt: Function;
export let stopResponse: Function;
......@@ -16,6 +19,7 @@
let filesInputElement;
let promptsElement;
let documentsElement;
let inputFiles;
let dragged = false;
......@@ -115,12 +119,16 @@
onMount(() => {
const dropZone = document.querySelector('body');
dropZone?.addEventListener('dragover', (e) => {
const onDragOver = (e) => {
e.preventDefault();
dragged = true;
});
};
dropZone.addEventListener('drop', async (e) => {
const onDragLeave = () => {
dragged = false;
};
const onDrop = async (e) => {
e.preventDefault();
console.log(e);
......@@ -143,14 +151,7 @@
const file = inputFiles[0];
if (['image/gif', 'image/jpeg', 'image/png'].includes(file['type'])) {
reader.readAsDataURL(file);
} else if (
[
'application/pdf',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'text/plain',
'text/csv'
].includes(file['type'])
) {
} else if (SUPPORTED_FILE_TYPE.includes(file['type'])) {
uploadDoc(file);
} else {
toast.error(`Unsupported File Type '${file['type']}'.`);
......@@ -161,11 +162,17 @@
}
dragged = false;
});
};
dropZone?.addEventListener('dragleave', () => {
dragged = false;
});
dropZone?.addEventListener('dragover', onDragOver);
dropZone?.addEventListener('drop', onDrop);
dropZone?.addEventListener('dragleave', onDragLeave);
return () => {
dropZone?.removeEventListener('dragover', onDragOver);
dropZone?.removeEventListener('drop', onDrop);
dropZone?.removeEventListener('dragleave', onDragLeave);
};
});
</script>
......@@ -179,12 +186,7 @@
<div class="absolute rounded-xl w-full h-full backdrop-blur bg-gray-800/40 flex justify-center">
<div class="m-auto pt-64 flex flex-col justify-center">
<div class="max-w-md">
<div class=" text-center text-6xl mb-3">🗂️</div>
<div class="text-center dark:text-white text-2xl font-semibold z-50">Add Files</div>
<div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
Drop any files/images here to add to the conversation
</div>
<AddFilesPlaceholder />
</div>
</div>
</div>
......@@ -224,6 +226,22 @@
<div class="w-full">
{#if prompt.charAt(0) === '/'}
<Prompts bind:this={promptsElement} bind:prompt />
{:else if prompt.charAt(0) === '#'}
<Documents
bind:this={documentsElement}
bind:prompt
on:select={(e) => {
console.log(e);
files = [
...files,
{
type: 'doc',
...e.detail,
upload_status: true
}
];
}}
/>
{:else if messages.length == 0 && suggestionPrompts.length !== 0}
<Suggestions {suggestionPrompts} {submitPrompt} />
{/if}
......@@ -256,14 +274,7 @@
const file = inputFiles[0];
if (['image/gif', 'image/jpeg', 'image/png'].includes(file['type'])) {
reader.readAsDataURL(file);
} else if (
[
'application/pdf',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'text/plain',
'text/csv'
].includes(file['type'])
) {
} else if (SUPPORTED_FILE_TYPE.includes(file['type'])) {
uploadDoc(file);
filesInputElement.value = '';
} else {
......@@ -448,8 +459,10 @@
editButton?.click();
}
if (prompt.charAt(0) === '/' && e.key === 'ArrowUp') {
promptsElement.selectUp();
if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'ArrowUp') {
e.preventDefault();
(promptsElement || documentsElement).selectUp();
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
......@@ -457,8 +470,10 @@
commandOptionButton.scrollIntoView({ block: 'center' });
}
if (prompt.charAt(0) === '/' && e.key === 'ArrowDown') {
promptsElement.selectDown();
if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'ArrowDown') {
e.preventDefault();
(promptsElement || documentsElement).selectDown();
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
......@@ -466,7 +481,7 @@
commandOptionButton.scrollIntoView({ block: 'center' });
}
if (prompt.charAt(0) === '/' && e.key === 'Enter') {
if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'Enter') {
e.preventDefault();
const commandOptionButton = [
......@@ -476,7 +491,7 @@
commandOptionButton?.click();
}
if (prompt.charAt(0) === '/' && e.key === 'Tab') {
if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'Tab') {
e.preventDefault();
const commandOptionButton = [
......
src/lib/components/chat/MessageInput/Documents.svelte 0 → 100644
View file @ f6a5d4b0
<script lang="ts">
import { createEventDispatcher } from 'svelte';
import { documents } from '$lib/stores';
import { removeFirstHashWord } from '$lib/utils';
import { tick } from 'svelte';
export let prompt = '';
const dispatch = createEventDispatcher();
let selectedIdx = 0;
let filteredDocs = [];
$: filteredDocs = $documents
.filter((p) => p.name.includes(prompt.split(' ')?.at(0)?.substring(1) ?? ''))
.sort((a, b) => a.title.localeCompare(b.title));
$: if (prompt) {
selectedIdx = 0;
}
export const selectUp = () => {
selectedIdx = Math.max(0, selectedIdx - 1);
};
export const selectDown = () => {
selectedIdx = Math.min(selectedIdx + 1, filteredDocs.length - 1);
};
const confirmSelect = async (doc) => {
dispatch('select', doc);
prompt = removeFirstHashWord(prompt);
const chatInputElement = document.getElementById('chat-textarea');
await tick();
chatInputElement?.focus();
await tick();
};
</script>
{#if filteredDocs.length > 0}
<div class="md:px-2 mb-3 text-left w-full">
<div class="flex w-full rounded-lg border border-gray-100 dark:border-gray-700">
<div class=" bg-gray-100 dark:bg-gray-700 w-10 rounded-l-lg text-center">
<div class=" text-lg font-semibold mt-2">#</div>
</div>
<div class="max-h-60 flex flex-col w-full rounded-r-lg">
<div class=" overflow-y-auto bg-white p-2 rounded-tr-lg space-y-0.5">
{#each filteredDocs as doc, docIdx}
<button
class=" px-3 py-1.5 rounded-lg w-full text-left {docIdx === selectedIdx
? ' bg-gray-100 selected-command-option-button'
: ''}"
type="button"
on:click={() => {
confirmSelect(doc);
}}
on:mousemove={() => {
selectedIdx = docIdx;
}}
on:focus={() => {}}
>
<div class=" font-medium text-black line-clamp-1">
#{doc.name} ({doc.filename})
</div>
<div class=" text-xs text-gray-600 line-clamp-1">
{doc.title}
</div>
</button>
{/each}
</div>
</div>
</div>
</div>
{/if}
src/lib/components/documents/EditDocModal.svelte 0 → 100644
View file @ f6a5d4b0
<script lang="ts">
import toast from 'svelte-french-toast';
import dayjs from 'dayjs';
import { onMount } from 'svelte';
import { getDocs, updateDocByName } from '$lib/apis/documents';
import Modal from '../common/Modal.svelte';
import { documents } from '$lib/stores';
export let show = false;
export let selectedDoc;
let doc = {
name: '',
title: ''
};
const submitHandler = async () => {
const res = await updateDocByName(localStorage.token, selectedDoc.name, {
title: doc.title,
name: doc.name
}).catch((error) => {
toast.error(error);
});
if (res) {
show = false;
documents.set(await getDocs(localStorage.token));
}
};
onMount(() => {
if (selectedDoc) {
doc = JSON.parse(JSON.stringify(selectedDoc));
}
});
</script>
<Modal size="sm" bind:show>
<div>
<div class=" flex justify-between dark:text-gray-300 px-5 py-4">
<div class=" text-lg font-medium self-center">Edit Doc</div>
<button
class="self-center"
on:click={() => {
show = false;
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="w-5 h-5"
>
<path
d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
/>
</svg>
</button>
</div>
<hr class=" dark:border-gray-800" />
<div class="flex flex-col md:flex-row w-full px-5 py-4 md:space-x-4 dark:text-gray-200">
<div class=" flex flex-col w-full sm:flex-row sm:justify-center sm:space-x-6">
<form
class="flex flex-col w-full"
on:submit|preventDefault={() => {
submitHandler();
}}
>
<div class=" flex flex-col space-y-1.5">
<div class="flex flex-col w-full">
<div class=" mb-1 text-xs text-gray-500">Name Tag</div>
<div class="flex flex-1">
<div
class="bg-gray-200 dark:bg-gray-600 font-bold px-3 py-1 border border-r-0 dark:border-gray-600 rounded-l-lg flex items-center"
>
#
</div>
<input
class="w-full rounded-r-lg py-2.5 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 disabled:text-gray-500 dark:disabled:text-gray-500 outline-none"
type="text"
bind:value={doc.name}
autocomplete="off"
required
/>
</div>
<!-- <div class="flex-1">
<input
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 disabled:text-gray-500 dark:disabled:text-gray-500 outline-none"
type="text"
bind:value={doc.name}
autocomplete="off"
required
/>
</div> -->
</div>
<div class="flex flex-col w-full">
<div class=" mb-1 text-xs text-gray-500">Title</div>
<div class="flex-1">
<input
class="w-full rounded-lg py-2.5 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
type="text"
bind:value={doc.title}
autocomplete="off"
required
/>
</div>
</div>
</div>
<div class="flex justify-end pt-5 text-sm font-medium">
<button
class=" px-4 py-2 bg-emerald-600 hover:bg-emerald-700 text-gray-100 transition rounded"
type="submit"
>
Save
</button>
</div>
</form>
</div>
</div>
</div>
</Modal>
<style>
input::-webkit-outer-spin-button,
input::-webkit-inner-spin-button {
/* display: none; <- Crashes Chrome on hover */
-webkit-appearance: none;
margin: 0; /* <-- Apparently some margin are still there even though it's hidden */
}
.tabs::-webkit-scrollbar {
display: none; /* for Chrome, Safari and Opera */
}
.tabs {
-ms-overflow-style: none; /* IE and Edge */
scrollbar-width: none; /* Firefox */
}
input[type='number'] {
-moz-appearance: textfield; /* Firefox */
}
</style>
src/lib/components/layout/Sidebar.svelte
View file @ f6a5d4b0
......@@ -68,7 +68,7 @@
<div class="px-2.5 flex justify-center space-x-2">
<button
id="sidebar-new-chat-button"
class="flex-grow flex justify-between rounded-md px-3 py-1.5 mt-2 hover:bg-gray-900 transition"
class="flex-grow flex justify-between rounded-md px-3 py-2 mt-1 hover:bg-gray-900 transition"
on:click={async () => {
goto('/');
......@@ -106,7 +106,7 @@
</div>
{#if $user?.role === 'admin'}
<div class="px-2.5 flex justify-center mt-1">
<div class="px-2.5 flex justify-center mt-0.5">
<button
class="flex-grow flex space-x-3 rounded-md px-3 py-2 hover:bg-gray-900 transition"
on:click={async () => {
......@@ -125,7 +125,7 @@
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M13.5 16.875h3.375m0 0h3.375m-3.375 0V13.5m0 3.375v3.375M6 10.5h2.25a2.25 2.25 0 002.25-2.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v2.25A2.25 2.25 0 006 10.5zm0 9.75h2.25A2.25 2.25 0 0010.5 18v-2.25a2.25 2.25 0 00-2.25-2.25H6a2.25 2.25 0 00-2.25 2.25V18A2.25 2.25 0 006 20.25zm9.75-9.75H18a2.25 2.25 0 002.25-2.25V6A2.25 2.25 0 0018 3.75h-2.25A2.25 2.25 0 0013.5 6v2.25a2.25 2.25 0 002.25 2.25z"
d="M13.5 16.875h3.375m0 0h3.375m-3.375 0V13.5m0 3.375v3.375M6 10.5h2.25a2.25 2.25 0 0 0 2.25-2.25V6a2.25 2.25 0 0 0-2.25-2.25H6A2.25 2.25 0 0 0 3.75 6v2.25A2.25 2.25 0 0 0 6 10.5Zm0 9.75h2.25A2.25 2.25 0 0 0 10.5 18v-2.25a2.25 2.25 0 0 0-2.25-2.25H6a2.25 2.25 0 0 0-2.25 2.25V18A2.25 2.25 0 0 0 6 20.25Zm9.75-9.75H18a2.25 2.25 0 0 0 2.25-2.25V6A2.25 2.25 0 0 0 18 3.75h-2.25A2.25 2.25 0 0 0 13.5 6v2.25a2.25 2.25 0 0 0 2.25 2.25Z"
/>
</svg>
</div>
......@@ -136,7 +136,7 @@
</button>
</div>
<div class="px-2.5 flex justify-center mb-1">
<div class="px-2.5 flex justify-center">
<button
class="flex-grow flex space-x-3 rounded-md px-3 py-2 hover:bg-gray-900 transition"
on:click={async () => {
......@@ -146,14 +146,16 @@
<div class="self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M11.013 2.513a1.75 1.75 0 0 1 2.475 2.474L6.226 12.25a2.751 2.751 0 0 1-.892.596l-2.047.848a.75.75 0 0 1-.98-.98l.848-2.047a2.75 2.75 0 0 1 .596-.892l7.262-7.261Z"
clip-rule="evenodd"
stroke-linecap="round"
stroke-linejoin="round"
d="m16.862 4.487 1.687-1.688a1.875 1.875 0 1 1 2.652 2.652L6.832 19.82a4.5 4.5 0 0 1-1.897 1.13l-2.685.8.8-2.685a4.5 4.5 0 0 1 1.13-1.897L16.863 4.487Zm0 0L19.5 7.125"
/>
</svg>
</div>
......@@ -163,6 +165,36 @@
</div>
</button>
</div>
<div class="px-2.5 flex justify-center mb-1">
<button
class="flex-grow flex space-x-3 rounded-md px-3 py-2 hover:bg-gray-900 transition"
on:click={async () => {
goto('/documents');
}}
>
<div class="self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="w-4 h-4"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M15.75 17.25v3.375c0 .621-.504 1.125-1.125 1.125h-9.75a1.125 1.125 0 0 1-1.125-1.125V7.875c0-.621.504-1.125 1.125-1.125H6.75a9.06 9.06 0 0 1 1.5.124m7.5 10.376h3.375c.621 0 1.125-.504 1.125-1.125V11.25c0-4.46-3.243-8.161-7.5-8.876a9.06 9.06 0 0 0-1.5-.124H9.375c-.621 0-1.125.504-1.125 1.125v3.5m7.5 10.375H9.375a1.125 1.125 0 0 1-1.125-1.125v-9.25m12 6.625v-1.875a3.375 3.375 0 0 0-3.375-3.375h-1.5a1.125 1.125 0 0 1-1.125-1.125v-1.5a3.375 3.375 0 0 0-3.375-3.375H9.75"
/>
</svg>
</div>
<div class="flex self-center">
<div class=" self-center font-medium text-sm">Documents</div>
</div>
</button>
</div>
{/if}
<div class="relative flex flex-col flex-1 overflow-y-auto">
......
src/lib/constants.ts
View file @ f6a5d4b0
......@@ -11,6 +11,13 @@ export const WEB_UI_VERSION = 'v1.0.0-alpha-static';
export const REQUIRED_OLLAMA_VERSION = '0.1.16';
export const SUPPORTED_FILE_TYPE = [
'application/pdf',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'text/plain',
'text/csv'
];
// Source: https://kit.svelte.dev/docs/modules#$env-static-public
// This feature, akin to $env/static/private, exclusively incorporates environment variables
// that are prefixed with config.kit.env.publicPrefix (usually set to PUBLIC_).
......
src/lib/stores/index.ts
View file @ f6a5d4b0
......@@ -11,8 +11,23 @@ export const chatId = writable('');
export const chats = writable([]);
export const models = writable([]);
export const modelfiles = writable([]);
export const prompts = writable([]);
export const documents = writable([
{
collection_name: 'collection_name',
filename: 'filename',
name: 'name',
title: 'title'
},
{
collection_name: 'collection_name1',
filename: 'filename1',
name: 'name1',
title: 'title1'
}
]);
export const settings = writable({});
export const showSettings = writable(false);
src/lib/utils/index.ts
View file @ f6a5d4b0
......@@ -128,6 +128,37 @@ export const findWordIndices = (text) => {
return matches;
};
export const removeFirstHashWord = (inputString) => {
// Split the string into an array of words
const words = inputString.split(' ');
// Find the index of the first word that starts with #
const index = words.findIndex((word) => word.startsWith('#'));
// Remove the first word with #
if (index !== -1) {
words.splice(index, 1);
}
// Join the remaining words back into a string
const resultString = words.join(' ');
return resultString;
};
export const transformFileName = (fileName) => {
// Convert to lowercase
const lowerCaseFileName = fileName.toLowerCase();
// Remove special characters using regular expression
const sanitizedFileName = lowerCaseFileName.replace(/[^\w\s]/g, '');
// Replace spaces with dashes
const finalFileName = sanitizedFileName.replace(/\s+/g, '-');
return finalFileName;
};
export const calculateSHA256 = async (file) => {
// Create a FileReader to read the file asynchronously
const reader = new FileReader();
......
src/routes/(app)/+layout.svelte
View file @ f6a5d4b0
......@@ -13,13 +13,22 @@
import { getOpenAIModels } from '$lib/apis/openai';
import { user, showSettings, settings, models, modelfiles, prompts } from '$lib/stores';
import {
user,
showSettings,
settings,
models,
modelfiles,
prompts,
documents
} from '$lib/stores';
import { REQUIRED_OLLAMA_VERSION, WEBUI_API_BASE_URL } from '$lib/constants';
import SettingsModal from '$lib/components/chat/SettingsModal.svelte';
import Sidebar from '$lib/components/layout/Sidebar.svelte';
import { checkVersion } from '$lib/utils';
import ShortcutsModal from '$lib/components/chat/ShortcutsModal.svelte';
import { getDocs } from '$lib/apis/documents';
let ollamaVersion = '';
let loaded = false;
......@@ -93,11 +102,10 @@
console.log();
await settings.set(JSON.parse(localStorage.getItem('settings') ?? '{}'));
await modelfiles.set(await getModelfiles(localStorage.token));
await modelfiles.set(await getModelfiles(localStorage.token));
await prompts.set(await getPrompts(localStorage.token));
console.log($modelfiles);
await documents.set(await getDocs(localStorage.token));
modelfiles.subscribe(async () => {
// should fetch models
......
src/routes/(app)/documents/+page.svelte 0 → 100644
View file @ f6a5d4b0
<script lang="ts">
import toast from 'svelte-french-toast';
import fileSaver from 'file-saver';
const { saveAs } = fileSaver;
import { onMount } from 'svelte';
import { documents } from '$lib/stores';
import { createNewDoc, deleteDocByName, getDocs } from '$lib/apis/documents';
import { SUPPORTED_FILE_TYPE } from '$lib/constants';
import { uploadDocToVectorDB } from '$lib/apis/rag';
import { transformFileName } from '$lib/utils';
import EditDocModal from '$lib/components/documents/EditDocModal.svelte';
let importFiles = '';
let inputFiles = '';
let query = '';
let showEditDocModal = false;
let selectedDoc;
let dragged = false;
const deleteDoc = async (name) => {
await deleteDocByName(localStorage.token, name);
await documents.set(await getDocs(localStorage.token));
};
const uploadDoc = async (file) => {
const res = await uploadDocToVectorDB(localStorage.token, '', file).catch((error) => {
toast.error(error);
return null;
});
if (res) {
await createNewDoc(
localStorage.token,
res.collection_name,
res.filename,
transformFileName(res.filename),
res.filename
).catch((error) => {
toast.error(error);
return null;
});
await documents.set(await getDocs(localStorage.token));
}
};
const onDragOver = (e) => {
e.preventDefault();
dragged = true;
};
const onDragLeave = () => {
dragged = false;
};
const onDrop = async (e) => {
e.preventDefault();
console.log(e);
if (e.dataTransfer?.files) {
const inputFiles = e.dataTransfer?.files;
if (inputFiles && inputFiles.length > 0) {
const file = inputFiles[0];
if (SUPPORTED_FILE_TYPE.includes(file['type'])) {
uploadDoc(file);
} else {
toast.error(`Unsupported File Type '${file['type']}'.`);
}
} else {
toast.error(`File not found.`);
}
}
dragged = false;
};
</script>
{#key selectedDoc}
<EditDocModal bind:show={showEditDocModal} {selectedDoc} />
{/key}
<div class="min-h-screen w-full flex justify-center dark:text-white">
<div class=" py-2.5 flex flex-col justify-between w-full">
<div class="max-w-2xl mx-auto w-full px-3 md:px-0 my-10">
<div class="mb-6 flex justify-between items-center">
<div class=" text-2xl font-semibold self-center">My Documents</div>
</div>
<div class=" flex w-full space-x-2">
<div class="flex flex-1">
<div class=" self-center ml-1 mr-3">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M9 3.5a5.5 5.5 0 100 11 5.5 5.5 0 000-11zM2 9a7 7 0 1112.452 4.391l3.328 3.329a.75.75 0 11-1.06 1.06l-3.329-3.328A7 7 0 012 9z"
clip-rule="evenodd"
/>
</svg>
</div>
<input
class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-none bg-transparent"
bind:value={query}
placeholder="Search Document"
/>
</div>
<div>
<button
class=" px-2 py-2 rounded-xl border border-gray-200 dark:border-gray-600 hover:bg-gray-100 dark:bg-gray-800 dark:hover:bg-gray-700 transition font-medium text-sm flex items-center space-x-1"
on:click={() => {
document.getElementById('upload-doc-input')?.click();
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
/>
</svg>
</button>
</div>
</div>
<input
id="upload-doc-input"
bind:files={inputFiles}
type="file"
hidden
on:change={async (e) => {
if (inputFiles && inputFiles.length > 0) {
const file = inputFiles[0];
if (SUPPORTED_FILE_TYPE.includes(file['type'])) {
uploadDoc(file);
} else {
toast.error(`Unsupported File Type '${file['type']}'.`);
}
inputFiles = null;
e.target.value = '';
} else {
toast.error(`File not found.`);
}
}}
/>
<div>
<div
class="my-3 py-16 rounded-lg border-2 border-dashed dark:border-gray-600 {dragged &&
' dark:bg-gray-700'} "
role="region"
on:drop={onDrop}
on:dragover={onDragOver}
on:dragleave={onDragLeave}
>
<div class=" pointer-events-none">
<div class="text-center dark:text-white text-2xl font-semibold z-50">Add Files</div>
<div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
Drop any files here to add to my documents
</div>
</div>
</div>
</div>
{#each $documents.filter((p) => query === '' || p.name.includes(query)) as doc}
<hr class=" dark:border-gray-700 my-2.5" />
<div class=" flex space-x-4 cursor-pointer w-full mb-3">
<div class=" flex flex-1 space-x-4 cursor-pointer w-full">
<div class=" flex items-center space-x-3">
<div class="p-2.5 bg-red-400 text-white rounded-lg">
{#if doc}
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="w-6 h-6"
>
<path
fill-rule="evenodd"
d="M5.625 1.5c-1.036 0-1.875.84-1.875 1.875v17.25c0 1.035.84 1.875 1.875 1.875h12.75c1.035 0 1.875-.84 1.875-1.875V12.75A3.75 3.75 0 0 0 16.5 9h-1.875a1.875 1.875 0 0 1-1.875-1.875V5.25A3.75 3.75 0 0 0 9 1.5H5.625ZM7.5 15a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 7.5 15Zm.75 2.25a.75.75 0 0 0 0 1.5H12a.75.75 0 0 0 0-1.5H8.25Z"
clip-rule="evenodd"
/>
<path
d="M12.971 1.816A5.23 5.23 0 0 1 14.25 5.25v1.875c0 .207.168.375.375.375H16.5a5.23 5.23 0 0 1 3.434 1.279 9.768 9.768 0 0 0-6.963-6.963Z"
/>
</svg>
{:else}
<svg
class=" w-6 h-6 translate-y-[0.5px]"
fill="currentColor"
viewBox="0 0 24 24"
xmlns="http://www.w3.org/2000/svg"
><style>
.spinner_qM83 {
animation: spinner_8HQG 1.05s infinite;
}
.spinner_oXPr {
animation-delay: 0.1s;
}
.spinner_ZTLf {
animation-delay: 0.2s;
}
@keyframes spinner_8HQG {
0%,
57.14% {
animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
transform: translate(0);
}
28.57% {
animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
transform: translateY(-6px);
}
100% {
transform: translate(0);
}
}
</style><circle class="spinner_qM83" cx="4" cy="12" r="2.5" /><circle
class="spinner_qM83 spinner_oXPr"
cx="12"
cy="12"
r="2.5"
/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="2.5" /></svg
>
{/if}
</div>
<div class=" flex-1 self-center flex-1">
<div class=" font-bold line-clamp-1">#{doc.name} ({doc.filename})</div>
<div class=" text-xs overflow-hidden text-ellipsis line-clamp-1">
{doc.title}
</div>
</div>
</div>
</div>
<div class="flex flex-row space-x-1 self-center">
<button
class="self-center w-fit text-sm px-2 py-2 border dark:border-gray-600 rounded-xl"
type="button"
on:click={async () => {
showEditDocModal = !showEditDocModal;
selectedDoc = doc;
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="w-4 h-4"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M16.862 4.487l1.687-1.688a1.875 1.875 0 112.652 2.652L6.832 19.82a4.5 4.5 0 01-1.897 1.13l-2.685.8.8-2.685a4.5 4.5 0 011.13-1.897L16.863 4.487zm0 0L19.5 7.125"
/>
</svg>
</button>
<!-- <button
class="self-center w-fit text-sm px-2 py-2 border dark:border-gray-600 rounded-xl"
type="button"
on:click={() => {
console.log('download file');
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
d="M8.75 2.75a.75.75 0 0 0-1.5 0v5.69L5.03 6.22a.75.75 0 0 0-1.06 1.06l3.5 3.5a.75.75 0 0 0 1.06 0l3.5-3.5a.75.75 0 0 0-1.06-1.06L8.75 8.44V2.75Z"
/>
<path
d="M3.5 9.75a.75.75 0 0 0-1.5 0v1.5A2.75 2.75 0 0 0 4.75 14h6.5A2.75 2.75 0 0 0 14 11.25v-1.5a.75.75 0 0 0-1.5 0v1.5c0 .69-.56 1.25-1.25 1.25h-6.5c-.69 0-1.25-.56-1.25-1.25v-1.5Z"
/>
</svg>
</button> -->
<button
class="self-center w-fit text-sm px-2 py-2 border dark:border-gray-600 rounded-xl"
type="button"
on:click={() => {
deleteDoc(doc.name);
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="w-4 h-4"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M14.74 9l-.346 9m-4.788 0L9.26 9m9.968-3.21c.342.052.682.107 1.022.166m-1.022-.165L18.16 19.673a2.25 2.25 0 01-2.244 2.077H8.084a2.25 2.25 0 01-2.244-2.077L4.772 5.79m14.456 0a48.108 48.108 0 00-3.478-.397m-12 .562c.34-.059.68-.114 1.022-.165m0 0a48.11 48.11 0 013.478-.397m7.5 0v-.916c0-1.18-.91-2.164-2.09-2.201a51.964 51.964 0 00-3.32 0c-1.18.037-2.09 1.022-2.09 2.201v.916m7.5 0a48.667 48.667 0 00-7.5 0"
/>
</svg>
</button>
</div>
</div>
{/each}
{#if $documents.length != 0}
<hr class=" dark:border-gray-700 my-2.5" />
<div class=" flex justify-between w-full mb-3">
<div class="flex space-x-2">
<input
id="documents-import-input"
bind:files={importFiles}
type="file"
accept=".json"
hidden
on:change={() => {
console.log(importFiles);
const reader = new FileReader();
reader.onload = async (event) => {
const savedDocs = JSON.parse(event.target.result);
console.log(savedDocs);
for (const doc of savedDocs) {
await createNewDoc(
localStorage.token,
doc.collection_name,
doc.filename,
doc.name,
doc.title
).catch((error) => {
toast.error(error);
return null;
});
}
await documents.set(await getDocs(localStorage.token));
};
reader.readAsText(importFiles[0]);
}}
/>
<button
class="self-center w-fit text-sm px-3 py-1 border dark:border-gray-600 rounded-xl flex"
on:click={async () => {
document.getElementById('documents-import-input')?.click();
}}
>
<div class=" self-center mr-2 font-medium">Import Documents Mapping</div>
<div class=" self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M4 2a1.5 1.5 0 0 0-1.5 1.5v9A1.5 1.5 0 0 0 4 14h8a1.5 1.5 0 0 0 1.5-1.5V6.621a1.5 1.5 0 0 0-.44-1.06L9.94 2.439A1.5 1.5 0 0 0 8.878 2H4Zm4 9.5a.75.75 0 0 1-.75-.75V8.06l-.72.72a.75.75 0 0 1-1.06-1.06l2-2a.75.75 0 0 1 1.06 0l2 2a.75.75 0 1 1-1.06 1.06l-.72-.72v2.69a.75.75 0 0 1-.75.75Z"
clip-rule="evenodd"
/>
</svg>
</div>
</button>
<button
class="self-center w-fit text-sm px-3 py-1 border dark:border-gray-600 rounded-xl flex"
on:click={async () => {
let blob = new Blob([JSON.stringify($documents)], {
type: 'application/json'
});
saveAs(blob, `documents-mapping-export-${Date.now()}.json`);
}}
>
<div class=" self-center mr-2 font-medium">Export Documents Mapping</div>
<div class=" self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M4 2a1.5 1.5 0 0 0-1.5 1.5v9A1.5 1.5 0 0 0 4 14h8a1.5 1.5 0 0 0 1.5-1.5V6.621a1.5 1.5 0 0 0-.44-1.06L9.94 2.439A1.5 1.5 0 0 0 8.878 2H4Zm4 3.5a.75.75 0 0 1 .75.75v2.69l.72-.72a.75.75 0 1 1 1.06 1.06l-2 2a.75.75 0 0 1-1.06 0l-2-2a.75.75 0 0 1 1.06-1.06l.72.72V6.25A.75.75 0 0 1 8 5.5Z"
clip-rule="evenodd"
/>
</svg>
</div>
</button>
<!-- <button
on:click={() => {
loadDefaultPrompts();
}}
>
dd
</button> -->
</div>
</div>
{/if}
<div class="text-xs flex items-center space-x-1">
<div>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="w-3 h-3"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="m11.25 11.25.041-.02a.75.75 0 0 1 1.063.852l-.708 2.836a.75.75 0 0 0 1.063.853l.041-.021M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Zm-9-3.75h.008v.008H12V8.25Z"
/>
</svg>
</div>
<div class="line-clamp-1">
Tip: Use '#' in the prompt input to swiftly load and select your documents.
</div>
</div>
</div>
</div>
</div>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment