Skip to content

GitLab

Commit e539cf5c authored by Timothy J. Baek's avatar Timothy J. Baek
Browse files

feat: gguf upload

parent 6ea9f6e1
Hide whitespace changes
Inline Side-by-side
Showing with 451 additions and 90 deletions
backend/.gitignore
View file @ e539cf5c
__pycache__ __pycache__
.env .env
\ No newline at end of file _old
uploads
\ No newline at end of file
backend/apps/web/main.py
View file @ e539cf5c
from fastapi import FastAPI, Request, Depends, HTTPException from fastapi import FastAPI, Request, Depends, HTTPException
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from apps.web.routers import auths, users from apps.web.routers import auths, users, utils
from config import WEBUI_VERSION, WEBUI_AUTH from config import WEBUI_VERSION, WEBUI_AUTH
app = FastAPI() app = FastAPI()
...@@ -19,6 +19,7 @@ app.add_middleware( ...@@ -19,6 +19,7 @@ app.add_middleware(
app.include_router(auths.router, prefix="/auths", tags=["auths"]) app.include_router(auths.router, prefix="/auths", tags=["auths"])
app.include_router(users.router, prefix="/users", tags=["users"]) app.include_router(users.router, prefix="/users", tags=["users"])
app.include_router(utils.router, prefix="/utils", tags=["utils"])
@app.get("/") @app.get("/")
......
backend/apps/web/routers/utils.py 0 → 100644
View file @ e539cf5c
from fastapi import APIRouter, UploadFile, File, BackgroundTasks
from fastapi import Depends, HTTPException, status
from starlette.responses import StreamingResponse
from pydantic import BaseModel
from utils.misc import calculate_sha256
import requests
import os
import asyncio
import json
from config import OLLAMA_API_BASE_URL
router = APIRouter()
class UploadBlobForm(BaseModel):
filename: str
@router.post("/upload")
async def upload(file: UploadFile = File(...)):
os.makedirs("./uploads", exist_ok=True)
file_path = os.path.join("./uploads", file.filename)
def file_write_stream():
total = 0
total_size = file.size
chunk_size = 1024 * 1024
done = False
try:
with open(file_path, "wb") as f:
while True:
chunk = file.file.read(chunk_size)
if not chunk:
break
f.write(chunk)
total += len(chunk)
done = total_size == total
res = {
"total": total_size,
"uploaded": total,
}
yield f"data: {json.dumps(res)}\n\n"
if done:
with open(file_path, "rb") as f:
hashed = calculate_sha256(f)
f.seek(0)
file_data = f.read()
url = f"{OLLAMA_API_BASE_URL}/blobs/sha256:{hashed}"
response = requests.post(url, data=file_data)
if response.ok:
res = {
"done": done,
"blob": f"sha256:{hashed}",
}
os.remove(file_path)
yield f"data: {json.dumps(res)}\n\n"
else:
raise "Ollama: Could not create blob, Please try again."
except Exception as e:
res = {"error": str(e)}
yield f"data: {json.dumps(res)}\n\n"
return StreamingResponse(file_write_stream(), media_type="text/event-stream")
backend/utils/misc.py
View file @ e539cf5c
...@@ -13,3 +13,11 @@ def get_gravatar_url(email): ...@@ -13,3 +13,11 @@ def get_gravatar_url(email):
# Grab the actual image URL # Grab the actual image URL
return f"https://www.gravatar.com/avatar/{hash_hex}?d=mp" return f"https://www.gravatar.com/avatar/{hash_hex}?d=mp"
def calculate_sha256(file):
sha256 = hashlib.sha256()
# Read the file in chunks to efficiently handle large files
for chunk in iter(lambda: file.read(8192), b""):
sha256.update(chunk)
return sha256.hexdigest()
src/lib/components/chat/SettingsModal.svelte
View file @ e539cf5c
<script lang="ts"> <script lang="ts">
import Modal from '../common/Modal.svelte'; import Modal from '../common/Modal.svelte';
import { WEB_UI_VERSION, OLLAMA_API_BASE_URL } from '$lib/constants'; import {
WEB_UI_VERSION,
OLLAMA_API_BASE_URL,
WEBUI_API_BASE_URL,
WEBUI_BASE_URL
} from '$lib/constants';
import toast from 'svelte-french-toast'; import toast from 'svelte-french-toast';
import { onMount } from 'svelte'; import { onMount } from 'svelte';
import { config, info, models, settings, user } from '$lib/stores'; import { config, info, models, settings, user } from '$lib/stores';
import { splitStream, getGravatarURL } from '$lib/utils'; import { splitStream, getGravatarURL } from '$lib/utils';
import Advanced from './Settings/Advanced.svelte'; import Advanced from './Settings/Advanced.svelte';
import { stringify } from 'postcss';
export let show = false; export let show = false;
...@@ -45,6 +51,10 @@ ...@@ -45,6 +51,10 @@
// Models // Models
let modelTag = ''; let modelTag = '';
let modelInputFile = '';
let modelInputFileBlob = '';
let modelFileContent = `TEMPLATE """{{ .System }}\nUSER: {{ .Prompt }}\nASSSISTANT: """\nPARAMETER num_ctx 4096\nPARAMETER stop "</s>"\nPARAMETER stop "USER:"\nPARAMETER stop "ASSSISTANT:"`;
let deleteModelTag = ''; let deleteModelTag = '';
let digest = ''; let digest = '';
let pullProgress = null; let pullProgress = null;
...@@ -220,6 +230,170 @@ ...@@ -220,6 +230,170 @@
models.set(await getModels()); models.set(await getModels());
}; };
const calculateSHA256 = async (file) => {
console.log(file);
// Create a FileReader to read the file asynchronously
const reader = new FileReader();
// Define a promise to handle the file reading
const readFile = new Promise((resolve, reject) => {
reader.onload = () => resolve(reader.result);
reader.onerror = reject;
});
// Read the file as an ArrayBuffer
reader.readAsArrayBuffer(file);
try {
// Wait for the FileReader to finish reading the file
const buffer = await readFile;
// Convert the ArrayBuffer to a Uint8Array
const uint8Array = new Uint8Array(buffer);
// Calculate the SHA-256 hash using Web Crypto API
const hashBuffer = await crypto.subtle.digest('SHA-256', uint8Array);
// Convert the hash to a hexadecimal string
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join('');
return `sha256:${hashHex}`;
} catch (error) {
console.error('Error calculating SHA-256 hash:', error);
throw error;
}
};
const uploadModelHandler = async () => {
const file = modelInputFile[0];
const formData = new FormData();
formData.append('file', file);
let uploaded = false;
const res = await fetch(`${WEBUI_API_BASE_URL}/utils/upload`, {
method: 'POST',
headers: {
...($settings.authHeader && { Authorization: $settings.authHeader }),
...($user && { Authorization: `Bearer ${localStorage.token}` })
},
body: formData
}).catch((error) => {
console.log(error);
return null;
});
if (res && res.ok) {
const reader = res.body
.pipeThrough(new TextDecoderStream())
.pipeThrough(splitStream('\n'))
.getReader();
while (true) {
const { value, done } = await reader.read();
if (done) break;
try {
let lines = value.split('\n');
for (const line of lines) {
if (line !== '') {
let data = JSON.parse(line.replace(/^data: /, ''));
console.log(data);
if (data.error) {
throw data.error;
}
if (data.done) {
modelInputFileBlob = data.blob;
uploaded = true;
}
}
}
} catch (error) {
console.log(error);
}
}
}
if (uploaded) {
const res = await fetch(`${$settings?.API_BASE_URL ?? OLLAMA_API_BASE_URL}/create`, {
method: 'POST',
headers: {
'Content-Type': 'text/event-stream',
...($settings.authHeader && { Authorization: $settings.authHeader }),
...($user && { Authorization: `Bearer ${localStorage.token}` })
},
body: JSON.stringify({
name: `${file.name}:latest`,
modelfile: `FROM @${modelInputFileBlob}\n${modelFileContent}`
})
}).catch((err) => {
console.log(err);
return null;
});
if (res && res.ok) {
const reader = res.body
.pipeThrough(new TextDecoderStream())
.pipeThrough(splitStream('\n'))
.getReader();
while (true) {
const { value, done } = await reader.read();
if (done) break;
try {
let lines = value.split('\n');
for (const line of lines) {
if (line !== '') {
console.log(line);
let data = JSON.parse(line);
console.log(data);
if (data.error) {
throw data.error;
}
if (data.detail) {
throw data.detail;
}
if (data.status) {
if (
!data.digest &&
!data.status.includes('writing') &&
!data.status.includes('sha256')
) {
toast.success(data.status);
} else {
if (data.digest) {
digest = data.digest;
if (data.completed) {
pullProgress = Math.round((data.completed / data.total) * 1000) / 10;
} else {
pullProgress = 100;
}
}
}
}
}
}
} catch (error) {
console.log(error);
toast.error(error);
}
}
}
}
modelTag = '';
models.set(await getModels());
};
const deleteModelHandler = async () => { const deleteModelHandler = async () => {
const res = await fetch(`${API_BASE_URL}/delete`, { const res = await fetch(`${API_BASE_URL}/delete`, {
method: 'DELETE', method: 'DELETE',
...@@ -790,104 +964,201 @@ ...@@ -790,104 +964,201 @@
</div> </div>
</div> </div>
{:else if selectedTab === 'models'} {:else if selectedTab === 'models'}
<div class="flex flex-col space-y-3 text-sm mb-10"> <div class="flex flex-col h-full justify-between text-sm">
<div> <div class=" space-y-3 pr-1.5 overflow-y-scroll h-80">
<div class=" mb-2.5 text-sm font-medium">Pull a model</div> <div>
<div class="flex w-full"> <div class=" mb-2.5 text-sm font-medium">Pull a model from Ollama.ai</div>
<div class="flex-1 mr-2"> <div class="flex w-full">
<input <div class="flex-1 mr-2">
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none" <input
placeholder="Enter model tag (e.g. mistral:7b)" class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
bind:value={modelTag} placeholder="Enter model tag (e.g. mistral:7b)"
/> bind:value={modelTag}
/>
</div>
<button
class="px-3 text-gray-100 bg-emerald-600 hover:bg-emerald-700 rounded transition"
on:click={() => {
pullModelHandler();
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
d="M8.75 2.75a.75.75 0 0 0-1.5 0v5.69L5.03 6.22a.75.75 0 0 0-1.06 1.06l3.5 3.5a.75.75 0 0 0 1.06 0l3.5-3.5a.75.75 0 0 0-1.06-1.06L8.75 8.44V2.75Z"
/>
<path
d="M3.5 9.75a.75.75 0 0 0-1.5 0v1.5A2.75 2.75 0 0 0 4.75 14h6.5A2.75 2.75 0 0 0 14 11.25v-1.5a.75.75 0 0 0-1.5 0v1.5c0 .69-.56 1.25-1.25 1.25h-6.5c-.69 0-1.25-.56-1.25-1.25v-1.5Z"
/>
</svg>
</button>
</div> </div>
<button
class="px-3 text-gray-100 bg-emerald-600 hover:bg-emerald-700 rounded transition" <div class="mt-2 text-xs text-gray-400 dark:text-gray-500">
on:click={() => { To access the available model names for downloading, <a
pullModelHandler(); class=" text-gray-500 dark:text-gray-300 font-medium"
}} href="https://ollama.ai/library"
> target="_blank">click here.</a
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="w-4 h-4"
> >
<path </div>
d="M10.75 2.75a.75.75 0 00-1.5 0v8.614L6.295 8.235a.75.75 0 10-1.09 1.03l4.25 4.5a.75.75 0 001.09 0l4.25-4.5a.75.75 0 00-1.09-1.03l-2.955 3.129V2.75z"
/>
<path
d="M3.5 12.75a.75.75 0 00-1.5 0v2.5A2.75 2.75 0 004.75 18h10.5A2.75 2.75 0 0018 15.25v-2.5a.75.75 0 00-1.5 0v2.5c0 .69-.56 1.25-1.25 1.25H4.75c-.69 0-1.25-.56-1.25-1.25v-2.5z"
/>
</svg>
</button>
</div>
<div class="mt-2 text-xs text-gray-400 dark:text-gray-500"> {#if pullProgress !== null}
To access the available model names for downloading, <a <div class="mt-2">
class=" text-gray-500 dark:text-gray-300 font-medium" <div class=" mb-2 text-xs">Pull Progress</div>
href="https://ollama.ai/library" <div class="w-full rounded-full dark:bg-gray-800">
target="_blank">click here.</a <div
> class="dark:bg-gray-600 text-xs font-medium text-blue-100 text-center p-0.5 leading-none rounded-full"
style="width: {Math.max(15, pullProgress ?? 0)}%"
>
{pullProgress ?? 0}%
</div>
</div>
<div class="mt-1 text-xs dark:text-gray-500" style="font-size: 0.5rem;">
{digest}
</div>
</div>
{/if}
</div> </div>
<hr class=" dark:border-gray-700" />
{#if pullProgress !== null} <div>
<div class="mt-2"> <div class=" mb-2.5 text-sm font-medium">Upload a GGUF model</div>
<div class=" mb-2 text-xs">Pull Progress</div> <div class="flex w-full mb-1.5">
<div class="w-full rounded-full dark:bg-gray-800"> <div class="flex-1 {modelInputFile && modelInputFile.length > 0 ? 'mr-2' : ''}">
<div <input
class="dark:bg-gray-600 text-xs font-medium text-blue-100 text-center p-0.5 leading-none rounded-full" id="model-upload-input"
style="width: {Math.max(15, pullProgress ?? 0)}%" type="file"
bind:files={modelInputFile}
on:change={() => {
console.log(modelInputFile);
}}
hidden
/>
<button
type="button"
class="w-full rounded text-left py-2 px-4 dark:text-gray-300 dark:bg-gray-800"
on:click={() => {
document.getElementById('model-upload-input').click();
}}
> >
{pullProgress ?? 0}% {#if modelInputFile && modelInputFile.length > 0}
</div> {modelInputFile[0].name}
</div> {:else}
<div class="mt-1 text-xs dark:text-gray-500" style="font-size: 0.5rem;"> Click here to select
{digest} {/if}
</button>
</div> </div>
{#if modelInputFile && modelInputFile.length > 0}
<button
class="px-3 text-gray-100 bg-emerald-600 hover:bg-emerald-700 rounded transition"
on:click={() => {
uploadModelHandler();
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
d="M7.25 10.25a.75.75 0 0 0 1.5 0V4.56l2.22 2.22a.75.75 0 1 0 1.06-1.06l-3.5-3.5a.75.75 0 0 0-1.06 0l-3.5 3.5a.75.75 0 0 0 1.06 1.06l2.22-2.22v5.69Z"
/>
<path
d="M3.5 9.75a.75.75 0 0 0-1.5 0v1.5A2.75 2.75 0 0 0 4.75 14h6.5A2.75 2.75 0 0 0 14 11.25v-1.5a.75.75 0 0 0-1.5 0v1.5c0 .69-.56 1.25-1.25 1.25h-6.5c-.69 0-1.25-.56-1.25-1.25v-1.5Z"
/>
</svg>
</button>
{/if}
</div> </div>
{/if}
</div>
<hr class=" dark:border-gray-700" />
<div> {#if modelInputFile && modelInputFile.length > 0}
<div class=" mb-2.5 text-sm font-medium">Delete a model</div> <div>
<div class="flex w-full"> <div>
<div class="flex-1 mr-2"> <div class=" my-2.5 text-sm font-medium">Modelfile Content</div>
<select <textarea
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none" bind:value={modelFileContent}
bind:value={deleteModelTag} class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none resize-none"
placeholder="Select a model" rows="6"
/>
</div>
</div>
{/if}
<div class=" mt-1 text-xs text-gray-400 dark:text-gray-500">
To access the GGUF models available for downloading, <a
class=" text-gray-500 dark:text-gray-300 font-medium"
href="https://huggingface.co/models?search=gguf"
target="_blank">click here.</a
> >
{#if !deleteModelTag}
<option value="" disabled selected>Select a model</option>
{/if}
{#each $models.filter((m) => m.size != null) as model}
<option value={model.name} class="bg-gray-100 dark:bg-gray-700"
>{model.name + ' (' + (model.size / 1024 ** 3).toFixed(1) + ' GB)'}</option
>
{/each}
</select>
</div> </div>
<button
class="px-3 bg-red-700 hover:bg-red-800 text-gray-100 rounded transition" {#if pullProgress !== null}
on:click={() => { <div class="mt-2">
deleteModelHandler(); <div class=" mb-2 text-xs">Pull Progress</div>
}} <div class="w-full rounded-full dark:bg-gray-800">
> <div
<svg class="dark:bg-gray-600 text-xs font-medium text-blue-100 text-center p-0.5 leading-none rounded-full"
xmlns="http://www.w3.org/2000/svg" style="width: {Math.max(15, pullProgress ?? 0)}%"
viewBox="0 0 20 20" >
fill="currentColor" {pullProgress ?? 0}%
class="w-4 h-4" </div>
</div>
<div class="mt-1 text-xs dark:text-gray-500" style="font-size: 0.5rem;">
{digest}
</div>
</div>
{/if}
</div>
<hr class=" dark:border-gray-700" />
<div>
<div class=" mb-2.5 text-sm font-medium">Delete a model</div>
<div class="flex w-full">
<div class="flex-1 mr-2">
<select
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
bind:value={deleteModelTag}
placeholder="Select a model"
>
{#if !deleteModelTag}
<option value="" disabled selected>Select a model</option>
{/if}
{#each $models.filter((m) => m.size != null) as model}
<option value={model.name} class="bg-gray-100 dark:bg-gray-700"
>{model.name +
' (' +
(model.size / 1024 ** 3).toFixed(1) +
' GB)'}</option
>
{/each}
</select>
</div>
<button
class="px-3 bg-red-700 hover:bg-red-800 text-gray-100 rounded transition"
on:click={() => {
deleteModelHandler();
}}
> >
<path <svg
fill-rule="evenodd" xmlns="http://www.w3.org/2000/svg"
d="M8.75 1A2.75 2.75 0 006 3.75v.443c-.795.077-1.584.176-2.365.298a.75.75 0 10.23 1.482l.149-.022.841 10.518A2.75 2.75 0 007.596 19h4.807a2.75 2.75 0 002.742-2.53l.841-10.52.149.023a.75.75 0 00.23-1.482A41.03 41.03 0 0014 4.193V3.75A2.75 2.75 0 0011.25 1h-2.5zM10 4c.84 0 1.673.025 2.5.075V3.75c0-.69-.56-1.25-1.25-1.25h-2.5c-.69 0-1.25.56-1.25 1.25v.325C8.327 4.025 9.16 4 10 4zM8.58 7.72a.75.75 0 00-1.5.06l.3 7.5a.75.75 0 101.5-.06l-.3-7.5zm4.34.06a.75.75 0 10-1.5-.06l-.3 7.5a.75.75 0 101.5.06l.3-7.5z" viewBox="0 0 16 16"
clip-rule="evenodd" fill="currentColor"
/> class="w-4 h-4"
</svg> >
</button> <path
fill-rule="evenodd"
d="M5 3.25V4H2.75a.75.75 0 0 0 0 1.5h.3l.815 8.15A1.5 1.5 0 0 0 5.357 15h5.285a1.5 1.5 0 0 0 1.493-1.35l.815-8.15h.3a.75.75 0 0 0 0-1.5H11v-.75A2.25 2.25 0 0 0 8.75 1h-1.5A2.25 2.25 0 0 0 5 3.25Zm2.25-.75a.75.75 0 0 0-.75.75V4h3v-.75a.75.75 0 0 0-.75-.75h-1.5ZM6.05 6a.75.75 0 0 1 .787.713l.275 5.5a.75.75 0 0 1-1.498.075l-.275-5.5A.75.75 0 0 1 6.05 6Zm3.9 0a.75.75 0 0 1 .712.787l-.275 5.5a.75.75 0 0 1-1.498-.075l.275-5.5a.75.75 0 0 1 .786-.711Z"
clip-rule="evenodd"
/>
</svg>
</button>
</div>
</div> </div>
</div> </div>
</div> </div>
......
src/lib/constants.ts
View file @ e539cf5c
...@@ -8,7 +8,8 @@ export const OLLAMA_API_BASE_URL = ...@@ -8,7 +8,8 @@ export const OLLAMA_API_BASE_URL =
: `http://localhost:11434/api` : `http://localhost:11434/api`
: PUBLIC_API_BASE_URL; : PUBLIC_API_BASE_URL;
export const WEBUI_API_BASE_URL = dev ? `http://${location.hostname}:8080/api/v1` : `/api/v1`; export const WEBUI_BASE_URL = dev ? `http://${location.hostname}:8080` : ``;
export const WEBUI_API_BASE_URL = `${WEBUI_BASE_URL}/api/v1`;
export const WEB_UI_VERSION = 'v1.0.0-alpha-static'; export const WEB_UI_VERSION = 'v1.0.0-alpha-static';
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment