Skip to content

GitLab

Commit 4ff17acc authored by Jun Siang Cheah's avatar Jun Siang Cheah
Browse files

Merge remote-tracking branch 'upstream/dev' into feat/oauth

parents f49d814d 9928114c
Hide whitespace changes
Inline Side-by-side
Showing with 2812 additions and 1139 deletions
src/lib/components/admin/Settings/Users.svelte
View file @ 4ff17acc
......@@ -48,7 +48,7 @@
await config.set(await getBackendConfig());
}}
>
<div class=" space-y-3 pr-1.5 overflow-y-scroll max-h-80">
<div class=" space-y-3 overflow-y-scroll max-h-full">
<div>
<div class=" mb-2 text-sm font-medium">{$i18n.t('User Permissions')}</div>
......@@ -58,11 +58,11 @@
<button
class="p-1 px-3 text-xs flex rounded transition"
on:click={() => {
permissions.chat.deletion = !permissions.chat.deletion;
permissions.chat.deletion = !(permissions?.chat?.deletion ?? true);
}}
type="button"
>
{#if permissions.chat.deletion}
{#if permissions?.chat?.deletion ?? true}
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
......@@ -94,7 +94,7 @@
</div>
</div>
<hr class=" dark:border-gray-700 my-2" />
<hr class=" dark:border-gray-850 my-2" />
<div class="mt-2 space-y-3">
<div>
......
src/lib/components/admin/Settings/WebSearch.svelte 0 → 100644
View file @ 4ff17acc
<script lang="ts">
import { getRAGConfig, updateRAGConfig } from '$lib/apis/rag';
import Switch from '$lib/components/common/Switch.svelte';
import { documents, models } from '$lib/stores';
import { onMount, getContext } from 'svelte';
import { toast } from 'svelte-sonner';
const i18n = getContext('i18n');
export let saveHandler: Function;
let webConfig = null;
let webSearchEngines = [
'searxng',
'google_pse',
'brave',
'serpstack',
'serper',
'serply',
'duckduckgo',
'tavily'
];
let youtubeLanguage = 'en';
let youtubeTranslation = null;
const submitHandler = async () => {
const res = await updateRAGConfig(localStorage.token, {
web: webConfig,
youtube: {
language: youtubeLanguage.split(',').map((lang) => lang.trim()),
translation: youtubeTranslation
}
});
};
onMount(async () => {
const res = await getRAGConfig(localStorage.token);
if (res) {
webConfig = res.web;
youtubeLanguage = res.youtube.language.join(',');
youtubeTranslation = res.youtube.translation;
}
});
</script>
<form
class="flex flex-col h-full justify-between space-y-3 text-sm"
on:submit|preventDefault={async () => {
await submitHandler();
saveHandler();
}}
>
<div class=" space-y-3 overflow-y-scroll scrollbar-hidden h-full">
{#if webConfig}
<div>
<div class=" mb-1 text-sm font-medium">
{$i18n.t('Web Search')}
</div>
<div>
<div class=" py-0.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
{$i18n.t('Enable Web Search')}
</div>
<Switch bind:state={webConfig.search.enabled} />
</div>
</div>
<div class=" py-0.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">{$i18n.t('Web Search Engine')}</div>
<div class="flex items-center relative">
<select
class="dark:bg-gray-900 w-fit pr-8 rounded px-2 p-1 text-xs bg-transparent outline-none text-right"
bind:value={webConfig.search.engine}
placeholder={$i18n.t('Select a engine')}
required
>
<option disabled selected value="">{$i18n.t('Select a engine')}</option>
{#each webSearchEngines as engine}
<option value={engine}>{engine}</option>
{/each}
</select>
</div>
</div>
{#if webConfig.search.engine !== ''}
<div class="mt-1.5">
{#if webConfig.search.engine === 'searxng'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Searxng Query URL')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
type="text"
placeholder={$i18n.t('Enter Searxng Query URL')}
bind:value={webConfig.search.searxng_query_url}
autocomplete="off"
/>
</div>
</div>
</div>
{:else if webConfig.search.engine === 'google_pse'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Google PSE API Key')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
type="text"
placeholder={$i18n.t('Enter Google PSE API Key')}
bind:value={webConfig.search.google_pse_api_key}
autocomplete="off"
/>
</div>
</div>
</div>
<div class="mt-1.5">
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Google PSE Engine Id')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
type="text"
placeholder={$i18n.t('Enter Google PSE Engine Id')}
bind:value={webConfig.search.google_pse_engine_id}
autocomplete="off"
/>
</div>
</div>
</div>
{:else if webConfig.search.engine === 'brave'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Brave Search API Key')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
type="text"
placeholder={$i18n.t('Enter Brave Search API Key')}
bind:value={webConfig.search.brave_search_api_key}
autocomplete="off"
/>
</div>
</div>
</div>
{:else if webConfig.search.engine === 'serpstack'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Serpstack API Key')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
type="text"
placeholder={$i18n.t('Enter Serpstack API Key')}
bind:value={webConfig.search.serpstack_api_key}
autocomplete="off"
/>
</div>
</div>
</div>
{:else if webConfig.search.engine === 'serper'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Serper API Key')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
type="text"
placeholder={$i18n.t('Enter Serper API Key')}
bind:value={webConfig.search.serper_api_key}
autocomplete="off"
/>
</div>
</div>
</div>
{:else if webConfig.search.engine === 'serply'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Serply API Key')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
type="text"
placeholder={$i18n.t('Enter Serply API Key')}
bind:value={webConfig.search.serply_api_key}
autocomplete="off"
/>
</div>
</div>
</div>
{:else if webConfig.search.engine === 'tavily'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Tavily API Key')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
type="text"
placeholder={$i18n.t('Enter Tavily API Key')}
bind:value={webConfig.search.tavily_api_key}
autocomplete="off"
/>
</div>
</div>
</div>
{/if}
</div>
{/if}
{#if webConfig.search.enabled}
<div class="mt-2 flex gap-2 mb-1">
<div class="w-full">
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Search Result Count')}
</div>
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
placeholder={$i18n.t('Search Result Count')}
bind:value={webConfig.search.result_count}
required
/>
</div>
<div class="w-full">
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Concurrent Requests')}
</div>
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
placeholder={$i18n.t('Concurrent Requests')}
bind:value={webConfig.search.concurrent_requests}
required
/>
</div>
</div>
{/if}
</div>
<hr class=" dark:border-gray-850 my-2" />
<div>
<div class=" mb-1 text-sm font-medium">
{$i18n.t('Web Loader Settings')}
</div>
<div>
<div class=" py-0.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
{$i18n.t('Bypass SSL verification for Websites')}
</div>
<button
class="p-1 px-3 text-xs flex rounded transition"
on:click={() => {
webConfig.ssl_verification = !webConfig.ssl_verification;
submitHandler();
}}
type="button"
>
{#if webConfig.ssl_verification === true}
<span class="ml-2 self-center">{$i18n.t('On')}</span>
{:else}
<span class="ml-2 self-center">{$i18n.t('Off')}</span>
{/if}
</button>
</div>
</div>
<div class=" mt-2 mb-1 text-sm font-medium">
{$i18n.t('Youtube Loader Settings')}
</div>
<div>
<div class=" py-0.5 flex w-full justify-between">
<div class=" w-20 text-xs font-medium self-center">{$i18n.t('Language')}</div>
<div class=" flex-1 self-center">
<input
class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
type="text"
placeholder={$i18n.t('Enter language codes')}
bind:value={youtubeLanguage}
autocomplete="off"
/>
</div>
</div>
</div>
</div>
{/if}
</div>
<div class="flex justify-end pt-3 text-sm font-medium">
<button
class=" px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg"
type="submit"
>
{$i18n.t('Save')}
</button>
</div>
</form>
src/lib/components/admin/SettingsModal.svelte
View file @ 4ff17acc
......@@ -39,181 +39,5 @@
</svg>
</button>
</div>
<div class="flex flex-col md:flex-row w-full p-4 md:space-x-4">
<div
class="tabs flex flex-row overflow-x-auto space-x-1 md:space-x-0 md:space-y-1 md:flex-col flex-1 md:flex-none md:w-40 dark:text-gray-200 text-xs text-left mb-3 md:mb-0"
>
<button
class="px-2.5 py-2.5 min-w-fit rounded-lg flex-1 md:flex-none flex text-right transition {selectedTab ===
'general'
? 'bg-gray-200 dark:bg-gray-700'
: ' hover:bg-gray-300 dark:hover:bg-gray-800'}"
on:click={() => {
selectedTab = 'general';
}}
>
<div class=" self-center mr-2">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M6.955 1.45A.5.5 0 0 1 7.452 1h1.096a.5.5 0 0 1 .497.45l.17 1.699c.484.12.94.312 1.356.562l1.321-1.081a.5.5 0 0 1 .67.033l.774.775a.5.5 0 0 1 .034.67l-1.08 1.32c.25.417.44.873.561 1.357l1.699.17a.5.5 0 0 1 .45.497v1.096a.5.5 0 0 1-.45.497l-1.699.17c-.12.484-.312.94-.562 1.356l1.082 1.322a.5.5 0 0 1-.034.67l-.774.774a.5.5 0 0 1-.67.033l-1.322-1.08c-.416.25-.872.44-1.356.561l-.17 1.699a.5.5 0 0 1-.497.45H7.452a.5.5 0 0 1-.497-.45l-.17-1.699a4.973 4.973 0 0 1-1.356-.562L4.108 13.37a.5.5 0 0 1-.67-.033l-.774-.775a.5.5 0 0 1-.034-.67l1.08-1.32a4.971 4.971 0 0 1-.561-1.357l-1.699-.17A.5.5 0 0 1 1 8.548V7.452a.5.5 0 0 1 .45-.497l1.699-.17c.12-.484.312-.94.562-1.356L2.629 4.107a.5.5 0 0 1 .034-.67l.774-.774a.5.5 0 0 1 .67-.033L5.43 3.71a4.97 4.97 0 0 1 1.356-.561l.17-1.699ZM6 8c0 .538.212 1.026.558 1.385l.057.057a2 2 0 0 0 2.828-2.828l-.058-.056A2 2 0 0 0 6 8Z"
clip-rule="evenodd"
/>
</svg>
</div>
<div class=" self-center">{$i18n.t('General')}</div>
</button>
<button
class="px-2.5 py-2.5 min-w-fit rounded-lg flex-1 md:flex-none flex text-right transition {selectedTab ===
'users'
? 'bg-gray-200 dark:bg-gray-700'
: ' hover:bg-gray-300 dark:hover:bg-gray-800'}"
on:click={() => {
selectedTab = 'users';
}}
>
<div class=" self-center mr-2">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
d="M8 8a2.5 2.5 0 1 0 0-5 2.5 2.5 0 0 0 0 5ZM3.156 11.763c.16-.629.44-1.21.813-1.72a2.5 2.5 0 0 0-2.725 1.377c-.136.287.102.58.418.58h1.449c.01-.077.025-.156.045-.237ZM12.847 11.763c.02.08.036.16.046.237h1.446c.316 0 .554-.293.417-.579a2.5 2.5 0 0 0-2.722-1.378c.374.51.653 1.09.813 1.72ZM14 7.5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0ZM3.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM5 13c-.552 0-1.013-.455-.876-.99a4.002 4.002 0 0 1 7.753 0c.136.535-.324.99-.877.99H5Z"
/>
</svg>
</div>
<div class=" self-center">{$i18n.t('Users')}</div>
</button>
<button
class="px-2.5 py-2.5 min-w-fit rounded-lg flex-1 md:flex-none flex text-right transition {selectedTab ===
'db'
? 'bg-gray-200 dark:bg-gray-700'
: ' hover:bg-gray-300 dark:hover:bg-gray-800'}"
on:click={() => {
selectedTab = 'db';
}}
>
<div class=" self-center mr-2">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path d="M8 7c3.314 0 6-1.343 6-3s-2.686-3-6-3-6 1.343-6 3 2.686 3 6 3Z" />
<path
d="M8 8.5c1.84 0 3.579-.37 4.914-1.037A6.33 6.33 0 0 0 14 6.78V8c0 1.657-2.686 3-6 3S2 9.657 2 8V6.78c.346.273.72.5 1.087.683C4.42 8.131 6.16 8.5 8 8.5Z"
/>
<path
d="M8 12.5c1.84 0 3.579-.37 4.914-1.037.366-.183.74-.41 1.086-.684V12c0 1.657-2.686 3-6 3s-6-1.343-6-3v-1.22c.346.273.72.5 1.087.683C4.42 12.131 6.16 12.5 8 12.5Z"
/>
</svg>
</div>
<div class=" self-center">{$i18n.t('Database')}</div>
</button>
<button
class="px-2.5 py-2.5 min-w-fit rounded-lg flex-1 md:flex-none flex text-right transition {selectedTab ===
'banners'
? 'bg-gray-200 dark:bg-gray-700'
: ' hover:bg-gray-300 dark:hover:bg-gray-800'}"
on:click={() => {
selectedTab = 'banners';
}}
>
<div class=" self-center mr-2">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-4"
>
<path
d="M5.85 3.5a.75.75 0 0 0-1.117-1 9.719 9.719 0 0 0-2.348 4.876.75.75 0 0 0 1.479.248A8.219 8.219 0 0 1 5.85 3.5ZM19.267 2.5a.75.75 0 1 0-1.118 1 8.22 8.22 0 0 1 1.987 4.124.75.75 0 0 0 1.48-.248A9.72 9.72 0 0 0 19.266 2.5Z"
/>
<path
fill-rule="evenodd"
d="M12 2.25A6.75 6.75 0 0 0 5.25 9v.75a8.217 8.217 0 0 1-2.119 5.52.75.75 0 0 0 .298 1.206c1.544.57 3.16.99 4.831 1.243a3.75 3.75 0 1 0 7.48 0 24.583 24.583 0 0 0 4.83-1.244.75.75 0 0 0 .298-1.205 8.217 8.217 0 0 1-2.118-5.52V9A6.75 6.75 0 0 0 12 2.25ZM9.75 18c0-.034 0-.067.002-.1a25.05 25.05 0 0 0 4.496 0l.002.1a2.25 2.25 0 1 1-4.5 0Z"
clip-rule="evenodd"
/>
</svg>
</div>
<div class=" self-center">{$i18n.t('Banners')}</div>
</button>
<button
class="px-2.5 py-2.5 min-w-fit rounded-lg flex-1 md:flex-none flex text-right transition {selectedTab ===
'pipelines'
? 'bg-gray-200 dark:bg-gray-700'
: ' hover:bg-gray-300 dark:hover:bg-gray-800'}"
on:click={() => {
selectedTab = 'pipelines';
}}
>
<div class=" self-center mr-2">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-4"
>
<path
d="M11.644 1.59a.75.75 0 0 1 .712 0l9.75 5.25a.75.75 0 0 1 0 1.32l-9.75 5.25a.75.75 0 0 1-.712 0l-9.75-5.25a.75.75 0 0 1 0-1.32l9.75-5.25Z"
/>
<path
d="m3.265 10.602 7.668 4.129a2.25 2.25 0 0 0 2.134 0l7.668-4.13 1.37.739a.75.75 0 0 1 0 1.32l-9.75 5.25a.75.75 0 0 1-.71 0l-9.75-5.25a.75.75 0 0 1 0-1.32l1.37-.738Z"
/>
<path
d="m10.933 19.231-7.668-4.13-1.37.739a.75.75 0 0 0 0 1.32l9.75 5.25c.221.12.489.12.71 0l9.75-5.25a.75.75 0 0 0 0-1.32l-1.37-.738-7.668 4.13a2.25 2.25 0 0 1-2.134-.001Z"
/>
</svg>
</div>
<div class=" self-center">{$i18n.t('Pipelines')}</div>
</button>
</div>
<div class="flex-1 md:min-h-[380px]">
{#if selectedTab === 'general'}
<General
saveHandler={() => {
toast.success($i18n.t('Settings saved successfully!'));
}}
/>
{:else if selectedTab === 'users'}
<Users
saveHandler={() => {
toast.success($i18n.t('Settings saved successfully!'));
}}
/>
{:else if selectedTab === 'db'}
<Database
saveHandler={() => {
toast.success($i18n.t('Settings saved successfully!'));
}}
/>
{:else if selectedTab === 'banners'}
<Banners
saveHandler={() => {
toast.success($i18n.t('Settings saved successfully!'));
}}
/>
{:else if selectedTab === 'pipelines'}
<Pipelines
saveHandler={() => {
toast.success($i18n.t('Settings saved successfully!'));
}}
/>
{/if}
</div>
</div>
</div>
</Modal>
src/lib/components/chat/Chat.svelte
View file @ 4ff17acc
......@@ -7,6 +7,10 @@
import { goto } from '$app/navigation';
import { page } from '$app/stores';
import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
import { OLLAMA_API_BASE_URL, OPENAI_API_BASE_URL, WEBUI_BASE_URL } from '$lib/constants';
import {
chatId,
chats,
......@@ -19,11 +23,14 @@
WEBUI_NAME,
banners,
user,
socket
socket,
showCallOverlay,
tools
} from '$lib/stores';
import {
convertMessagesToHistory,
copyToClipboard,
extractSentencesForAudio,
promptTemplate,
splitStream
} from '$lib/utils';
......@@ -39,30 +46,27 @@
getTagsById,
updateChatById
} from '$lib/apis/chats';
import {
generateOpenAIChatCompletion,
generateSearchQuery,
generateTitle
} from '$lib/apis/openai';
import { generateOpenAIChatCompletion } from '$lib/apis/openai';
import { runWebSearch } from '$lib/apis/rag';
import { createOpenAITextStream } from '$lib/apis/streaming';
import { queryMemory } from '$lib/apis/memories';
import { getUserSettings } from '$lib/apis/users';
import { chatCompleted, generateTitle, generateSearchQuery } from '$lib/apis';
import Banner from '../common/Banner.svelte';
import MessageInput from '$lib/components/chat/MessageInput.svelte';
import Messages from '$lib/components/chat/Messages.svelte';
import Navbar from '$lib/components/layout/Navbar.svelte';
import { OLLAMA_API_BASE_URL, OPENAI_API_BASE_URL, WEBUI_BASE_URL } from '$lib/constants';
import { createOpenAITextStream } from '$lib/apis/streaming';
import { queryMemory } from '$lib/apis/memories';
import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
import { runWebSearch } from '$lib/apis/rag';
import Banner from '../common/Banner.svelte';
import { getUserSettings } from '$lib/apis/users';
import { chatCompleted } from '$lib/apis';
import CallOverlay from './MessageInput/CallOverlay.svelte';
import { error } from '@sveltejs/kit';
const i18n: Writable<i18nType> = getContext('i18n');
export let chatIdProp = '';
let loaded = false;
const eventTarget = new EventTarget();
let stopResponseFlag = false;
let autoScroll = true;
let processing = '';
......@@ -73,6 +77,10 @@
let selectedModels = [''];
let atSelectedModel: Model | undefined;
let selectedModelIds = [];
$: selectedModelIds = atSelectedModel !== undefined ? [atSelectedModel.id] : selectedModels;
let selectedToolIds = [];
let webSearchEnabled = false;
let chat = null;
......@@ -103,7 +111,8 @@
$: if (chatIdProp) {
(async () => {
if (await loadChat()) {
console.log(chatIdProp);
if (chatIdProp && (await loadChat())) {
await tick();
loaded = true;
......@@ -118,7 +127,11 @@
onMount(async () => {
if (!$chatId) {
await initNewChat();
chatId.subscribe(async (value) => {
if (!value) {
await initNewChat();
}
});
} else {
if (!($settings.saveChatHistory ?? true)) {
await goto('/');
......@@ -292,10 +305,11 @@
};
//////////////////////////
// Ollama functions
// Chat functions
//////////////////////////
const submitPrompt = async (userPrompt, _user = null) => {
const submitPrompt = async (userPrompt, { _raw = false } = {}) => {
let _responses = [];
console.log('submitPrompt', $chatId);
selectedModels = selectedModels.map((modelId) =>
......@@ -318,8 +332,18 @@
)
);
} else {
// Reset chat message textarea height
document.getElementById('chat-textarea').style.height = '';
// Reset chat input textarea
const chatTextAreaElement = document.getElementById('chat-textarea');
if (chatTextAreaElement) {
chatTextAreaElement.value = '';
chatTextAreaElement.style.height = '';
}
const _files = JSON.parse(JSON.stringify(files));
files = [];
prompt = '';
// Create user message
let userMessageId = uuidv4();
......@@ -328,9 +352,8 @@
parentId: messages.length !== 0 ? messages.at(-1).id : null,
childrenIds: [],
role: 'user',
user: _user ?? undefined,
content: userPrompt,
files: files.length > 0 ? files : undefined,
files: _files.length > 0 ? _files : undefined,
timestamp: Math.floor(Date.now() / 1000), // Unix epoch
models: selectedModels.filter((m, mIdx) => selectedModels.indexOf(m) === mIdx)
};
......@@ -346,51 +369,86 @@
// Wait until history/message have been updated
await tick();
_responses = await sendPrompt(userPrompt, userMessageId, { newChat: true });
}
// Create new chat if only one message in messages
if (messages.length == 1) {
if ($settings.saveChatHistory ?? true) {
chat = await createNewChat(localStorage.token, {
id: $chatId,
title: $i18n.t('New Chat'),
models: selectedModels,
system: $settings.system ?? undefined,
options: {
...($settings.params ?? {})
},
messages: messages,
history: history,
tags: [],
timestamp: Date.now()
});
await chats.set(await getChatList(localStorage.token));
await chatId.set(chat.id);
} else {
await chatId.set('local');
return _responses;
};
const sendPrompt = async (prompt, parentId, { modelId = null, newChat = false } = {}) => {
let _responses = [];
// If modelId is provided, use it, else use selected model
let selectedModelIds = modelId
? [modelId]
: atSelectedModel !== undefined
? [atSelectedModel.id]
: selectedModels;
// Create response messages for each selected model
const responseMessageIds = {};
for (const modelId of selectedModelIds) {
const model = $models.filter((m) => m.id === modelId).at(0);
if (model) {
let responseMessageId = uuidv4();
let responseMessage = {
parentId: parentId,
id: responseMessageId,
childrenIds: [],
role: 'assistant',
content: '',
model: model.id,
modelName: model.name ?? model.id,
userContext: null,
timestamp: Math.floor(Date.now() / 1000) // Unix epoch
};
// Add message to history and Set currentId to messageId
history.messages[responseMessageId] = responseMessage;
history.currentId = responseMessageId;
// Append messageId to childrenIds of parent message
if (parentId !== null) {
history.messages[parentId].childrenIds = [
...history.messages[parentId].childrenIds,
responseMessageId
];
}
await tick();
}
// Reset chat input textarea
prompt = '';
document.getElementById('chat-textarea').style.height = '';
files = [];
responseMessageIds[modelId] = responseMessageId;
}
}
await tick();
// Send prompt
await sendPrompt(userPrompt, userMessageId);
// Create new chat if only one message in messages
if (newChat && messages.length == 2) {
if ($settings.saveChatHistory ?? true) {
chat = await createNewChat(localStorage.token, {
id: $chatId,
title: $i18n.t('New Chat'),
models: selectedModels,
system: $settings.system ?? undefined,
options: {
...($settings.params ?? {})
},
messages: messages,
history: history,
tags: [],
timestamp: Date.now()
});
await chats.set(await getChatList(localStorage.token));
await chatId.set(chat.id);
} else {
await chatId.set('local');
}
await tick();
}
};
const sendPrompt = async (prompt, parentId, modelId = null) => {
const _chatId = JSON.parse(JSON.stringify($chatId));
await Promise.all(
(modelId
? [modelId]
: atSelectedModel !== undefined
? [atSelectedModel.id]
: selectedModels
).map(async (modelId) => {
selectedModelIds.map(async (modelId) => {
console.log('modelId', modelId);
const model = $models.filter((m) => m.id === modelId).at(0);
......@@ -408,33 +466,8 @@
);
}
// Create response message
let responseMessageId = uuidv4();
let responseMessage = {
parentId: parentId,
id: responseMessageId,
childrenIds: [],
role: 'assistant',
content: '',
model: model.id,
modelName: model.name ?? model.id,
userContext: null,
timestamp: Math.floor(Date.now() / 1000) // Unix epoch
};
// Add message to history and Set currentId to messageId
history.messages[responseMessageId] = responseMessage;
history.currentId = responseMessageId;
// Append messageId to childrenIds of parent message
if (parentId !== null) {
history.messages[parentId].childrenIds = [
...history.messages[parentId].childrenIds,
responseMessageId
];
}
await tick();
let responseMessageId = responseMessageIds[modelId];
let responseMessage = history.messages[responseMessageId];
let userContext = null;
if ($settings?.memory ?? false) {
......@@ -443,7 +476,6 @@
toast.error(error);
return null;
});
if (res) {
if (res.documents[0].length > 0) {
userContext = res.documents.reduce((acc, doc, index) => {
......@@ -463,18 +495,17 @@
responseMessage.userContext = userContext;
const chatEventEmitter = await getChatEventEmitter(model.id, _chatId);
if (webSearchEnabled) {
await getWebSearchResults(model.id, parentId, responseMessageId);
}
let _response = null;
if (model?.owned_by === 'openai') {
await sendPromptOpenAI(model, prompt, responseMessageId, _chatId);
_response = await sendPromptOpenAI(model, prompt, responseMessageId, _chatId);
} else if (model) {
await sendPromptOllama(model, prompt, responseMessageId, _chatId);
_response = await sendPromptOllama(model, prompt, responseMessageId, _chatId);
}
console.log('chatEventEmitter', chatEventEmitter);
_responses.push(_response);
if (chatEventEmitter) clearInterval(chatEventEmitter);
} else {
......@@ -484,81 +515,11 @@
);
await chats.set(await getChatList(localStorage.token));
};
const getWebSearchResults = async (model: string, parentId: string, responseId: string) => {
const responseMessage = history.messages[responseId];
responseMessage.status = {
done: false,
action: 'web_search',
description: $i18n.t('Generating search query')
};
messages = messages;
const prompt = history.messages[parentId].content;
let searchQuery = prompt;
if (prompt.length > 100) {
searchQuery = await generateChatSearchQuery(model, prompt);
if (!searchQuery) {
toast.warning($i18n.t('No search query generated'));
responseMessage.status = {
...responseMessage.status,
done: true,
error: true,
description: 'No search query generated'
};
messages = messages;
return;
}
}
responseMessage.status = {
...responseMessage.status,
description: $i18n.t("Searching the web for '{{searchQuery}}'", { searchQuery })
};
messages = messages;
const results = await runWebSearch(localStorage.token, searchQuery).catch((error) => {
console.log(error);
toast.error(error);
return null;
});
if (results) {
responseMessage.status = {
...responseMessage.status,
done: true,
description: $i18n.t('Searched {{count}} sites', { count: results.filenames.length }),
urls: results.filenames
};
if (responseMessage?.files ?? undefined === undefined) {
responseMessage.files = [];
}
responseMessage.files.push({
collection_name: results.collection_name,
name: searchQuery,
type: 'web_search_results',
urls: results.filenames
});
messages = messages;
} else {
responseMessage.status = {
...responseMessage.status,
done: true,
error: true,
description: 'No search results found'
};
messages = messages;
}
return _responses;
};
const sendPromptOllama = async (model, userPrompt, responseMessageId, _chatId) => {
model = model.id;
let _response = null;
const responseMessage = history.messages[responseMessageId];
......@@ -617,17 +578,39 @@
}
});
const docs = messages
.filter((message) => message?.files ?? null)
.map((message) =>
message.files.filter((item) =>
['doc', 'collection', 'web_search_results'].includes(item.type)
let docs = [];
if (model?.info?.meta?.knowledge ?? false) {
docs = model.info.meta.knowledge;
}
docs = [
...docs,
...messages
.filter((message) => message?.files ?? null)
.map((message) =>
message.files.filter((item) =>
['doc', 'collection', 'web_search_results'].includes(item.type)
)
)
)
.flat(1);
.flat(1)
].filter(
(item, index, array) =>
array.findIndex((i) => JSON.stringify(i) === JSON.stringify(item)) === index
);
eventTarget.dispatchEvent(
new CustomEvent('chat:start', {
detail: {
id: responseMessageId
}
})
);
await tick();
const [res, controller] = await generateChatCompletion(localStorage.token, {
model: model,
model: model.id,
messages: messagesBody,
options: {
...($settings.params ?? {}),
......@@ -642,6 +625,7 @@
},
format: $settings.requestFormat ?? undefined,
keep_alive: $settings.keepAlive ?? undefined,
tool_ids: selectedToolIds.length > 0 ? selectedToolIds : undefined,
docs: docs.length > 0 ? docs : undefined,
citations: docs.length > 0,
chat_id: $chatId
......@@ -665,9 +649,10 @@
controller.abort('User: Stop Response');
} else {
const messages = createMessagesList(responseMessageId);
await chatCompletedHandler(model, messages);
await chatCompletedHandler(model.id, messages);
}
_response = responseMessage.content;
break;
}
......@@ -693,6 +678,23 @@
continue;
} else {
responseMessage.content += data.message.content;
const sentences = extractSentencesForAudio(responseMessage.content);
sentences.pop();
// dispatch only last sentence and make sure it hasn't been dispatched before
if (
sentences.length > 0 &&
sentences[sentences.length - 1] !== responseMessage.lastSentence
) {
responseMessage.lastSentence = sentences[sentences.length - 1];
eventTarget.dispatchEvent(
new CustomEvent('chat', {
detail: { id: responseMessageId, content: sentences[sentences.length - 1] }
})
);
}
messages = messages;
}
} else {
......@@ -719,25 +721,17 @@
messages = messages;
if ($settings.notificationEnabled && !document.hasFocus()) {
const notification = new Notification(
selectedModelfile
? `${
selectedModelfile.title.charAt(0).toUpperCase() +
selectedModelfile.title.slice(1)
}`
: `${model}`,
{
body: responseMessage.content,
icon: selectedModelfile?.imageUrl ?? `${WEBUI_BASE_URL}/static/favicon.png`
}
);
const notification = new Notification(`${model.id}`, {
body: responseMessage.content,
icon: `${WEBUI_BASE_URL}/static/favicon.png`
});
}
if ($settings.responseAutoCopy) {
if ($settings?.responseAutoCopy ?? false) {
copyToClipboard(responseMessage.content);
}
if ($settings.responseAutoPlayback) {
if ($settings.responseAutoPlayback && !$showCallOverlay) {
await tick();
document.getElementById(`speak-button-${responseMessage.id}`)?.click();
}
......@@ -795,6 +789,23 @@
stopResponseFlag = false;
await tick();
let lastSentence = extractSentencesForAudio(responseMessage.content)?.at(-1) ?? '';
if (lastSentence) {
eventTarget.dispatchEvent(
new CustomEvent('chat', {
detail: { id: responseMessageId, content: lastSentence }
})
);
}
eventTarget.dispatchEvent(
new CustomEvent('chat:finish', {
detail: {
id: responseMessageId,
content: responseMessage.content
}
})
);
if (autoScroll) {
scrollToBottom();
}
......@@ -804,24 +815,46 @@
const _title = await generateChatTitle(userPrompt);
await setChatTitle(_chatId, _title);
}
return _response;
};
const sendPromptOpenAI = async (model, userPrompt, responseMessageId, _chatId) => {
let _response = null;
const responseMessage = history.messages[responseMessageId];
const docs = messages
.filter((message) => message?.files ?? null)
.map((message) =>
message.files.filter((item) =>
['doc', 'collection', 'web_search_results'].includes(item.type)
)
)
.flat(1);
let docs = [];
console.log(docs);
if (model?.info?.meta?.knowledge ?? false) {
docs = model.info.meta.knowledge;
}
docs = [
...docs,
...messages
.filter((message) => message?.files ?? null)
.map((message) =>
message.files.filter((item) =>
['doc', 'collection', 'web_search_results'].includes(item.type)
)
)
.flat(1)
].filter(
(item, index, array) =>
array.findIndex((i) => JSON.stringify(i) === JSON.stringify(item)) === index
);
scrollToBottom();
eventTarget.dispatchEvent(
new CustomEvent('chat:start', {
detail: {
id: responseMessageId
}
})
);
await tick();
try {
const [res, controller] = await generateOpenAIChatCompletion(
localStorage.token,
......@@ -889,6 +922,7 @@
top_p: $settings?.params?.top_p ?? undefined,
frequency_penalty: $settings?.params?.frequency_penalty ?? undefined,
max_tokens: $settings?.params?.max_tokens ?? undefined,
tool_ids: selectedToolIds.length > 0 ? selectedToolIds : undefined,
docs: docs.length > 0 ? docs : undefined,
citations: docs.length > 0,
chat_id: $chatId
......@@ -923,6 +957,8 @@
await chatCompletedHandler(model.id, messages);
}
_response = responseMessage.content;
break;
}
......@@ -939,6 +975,23 @@
continue;
} else {
responseMessage.content += value;
const sentences = extractSentencesForAudio(responseMessage.content);
sentences.pop();
// dispatch only last sentence and make sure it hasn't been dispatched before
if (
sentences.length > 0 &&
sentences[sentences.length - 1] !== responseMessage.lastSentence
) {
responseMessage.lastSentence = sentences[sentences.length - 1];
eventTarget.dispatchEvent(
new CustomEvent('chat', {
detail: { id: responseMessageId, content: sentences[sentences.length - 1] }
})
);
}
messages = messages;
}
......@@ -948,7 +1001,7 @@
}
if ($settings.notificationEnabled && !document.hasFocus()) {
const notification = new Notification(`OpenAI ${model}`, {
const notification = new Notification(`${model.id}`, {
body: responseMessage.content,
icon: `${WEBUI_BASE_URL}/static/favicon.png`
});
......@@ -958,8 +1011,9 @@
copyToClipboard(responseMessage.content);
}
if ($settings.responseAutoPlayback) {
if ($settings.responseAutoPlayback && !$showCallOverlay) {
await tick();
document.getElementById(`speak-button-${responseMessage.id}`)?.click();
}
......@@ -988,6 +1042,24 @@
stopResponseFlag = false;
await tick();
let lastSentence = extractSentencesForAudio(responseMessage.content)?.at(-1) ?? '';
if (lastSentence) {
eventTarget.dispatchEvent(
new CustomEvent('chat', {
detail: { id: responseMessageId, content: lastSentence }
})
);
}
eventTarget.dispatchEvent(
new CustomEvent('chat:finish', {
detail: {
id: responseMessageId,
content: responseMessage.content
}
})
);
if (autoScroll) {
scrollToBottom();
}
......@@ -998,6 +1070,8 @@
const _title = await generateChatTitle(userPrompt);
await setChatTitle(_chatId, _title);
}
return _response;
};
const handleOpenAIError = async (error, res: Response | null, model, responseMessage) => {
......@@ -1052,9 +1126,12 @@
let userPrompt = userMessage.content;
if ((userMessage?.models ?? [...selectedModels]).length == 1) {
// If user message has only one model selected, sendPrompt automatically selects it for regeneration
await sendPrompt(userPrompt, userMessage.id);
} else {
await sendPrompt(userPrompt, userMessage.id, message.model);
// If there are multiple models selected, use the model of the response message for regeneration
// e.g. many model chat
await sendPrompt(userPrompt, userMessage.id, { modelId: message.model });
}
}
};
......@@ -1093,28 +1170,15 @@
const generateChatTitle = async (userPrompt) => {
if ($settings?.title?.auto ?? true) {
const model = $models.find((model) => model.id === selectedModels[0]);
const titleModelId =
model?.owned_by === 'openai' ?? false
? $settings?.title?.modelExternal ?? selectedModels[0]
: $settings?.title?.model ?? selectedModels[0];
const titleModel = $models.find((model) => model.id === titleModelId);
console.log(titleModel);
const title = await generateTitle(
localStorage.token,
$settings?.title?.prompt ??
$i18n.t(
"Create a concise, 3-5 word phrase as a header for the following query, strictly adhering to the 3-5 word limit and avoiding the use of the word 'title':"
) + ' {{prompt}}',
titleModelId,
selectedModels[0],
userPrompt,
$chatId,
titleModel?.owned_by === 'openai' ?? false
? `${OPENAI_API_BASE_URL}`
: `${OLLAMA_API_BASE_URL}/v1`
);
$chatId
).catch((error) => {
console.error(error);
return 'New Chat';
});
return title;
} else {
......@@ -1122,29 +1186,6 @@
}
};
const generateChatSearchQuery = async (modelId: string, prompt: string) => {
const model = $models.find((model) => model.id === modelId);
const taskModelId =
model?.owned_by === 'openai' ?? false
? $settings?.title?.modelExternal ?? modelId
: $settings?.title?.model ?? modelId;
const taskModel = $models.find((model) => model.id === taskModelId);
const previousMessages = messages
.filter((message) => message.role === 'user')
.map((message) => message.content);
return await generateSearchQuery(
localStorage.token,
taskModelId,
previousMessages,
prompt,
taskModel?.owned_by === 'openai' ?? false
? `${OPENAI_API_BASE_URL}`
: `${OLLAMA_API_BASE_URL}/v1`
);
};
const setChatTitle = async (_chatId, _title) => {
if (_chatId === $chatId) {
title = _title;
......@@ -1156,32 +1197,88 @@
}
};
const getTags = async () => {
return await getTagsById(localStorage.token, $chatId).catch(async (error) => {
return [];
const getWebSearchResults = async (model: string, parentId: string, responseId: string) => {
const responseMessage = history.messages[responseId];
responseMessage.statusHistory = [
{
done: false,
action: 'web_search',
description: $i18n.t('Generating search query')
}
];
messages = messages;
const prompt = history.messages[parentId].content;
let searchQuery = await generateSearchQuery(localStorage.token, model, messages, prompt).catch(
(error) => {
console.log(error);
return prompt;
}
);
if (!searchQuery) {
toast.warning($i18n.t('No search query generated'));
responseMessage.statusHistory.push({
done: true,
error: true,
action: 'web_search',
description: 'No search query generated'
});
messages = messages;
}
responseMessage.statusHistory.push({
done: false,
action: 'web_search',
description: $i18n.t(`Searching "{{searchQuery}}"`, { searchQuery })
});
};
messages = messages;
const addTag = async (tagName) => {
const res = await addTagById(localStorage.token, $chatId, tagName);
tags = await getTags();
const results = await runWebSearch(localStorage.token, searchQuery).catch((error) => {
console.log(error);
toast.error(error);
chat = await updateChatById(localStorage.token, $chatId, {
tags: tags
return null;
});
_tags.set(await getAllChatTags(localStorage.token));
};
if (results) {
responseMessage.statusHistory.push({
done: true,
action: 'web_search',
description: $i18n.t('Searched {{count}} sites', { count: results.filenames.length }),
query: searchQuery,
urls: results.filenames
});
const deleteTag = async (tagName) => {
const res = await deleteTagById(localStorage.token, $chatId, tagName);
tags = await getTags();
if (responseMessage?.files ?? undefined === undefined) {
responseMessage.files = [];
}
chat = await updateChatById(localStorage.token, $chatId, {
tags: tags
});
responseMessage.files.push({
collection_name: results.collection_name,
name: searchQuery,
type: 'web_search_results',
urls: results.filenames
});
messages = messages;
} else {
responseMessage.statusHistory.push({
done: true,
error: true,
action: 'web_search',
description: 'No search results found'
});
messages = messages;
}
};
_tags.set(await getAllChatTags(localStorage.token));
const getTags = async () => {
return await getTagsById(localStorage.token, $chatId).catch(async (error) => {
return [];
});
};
</script>
......@@ -1193,6 +1290,19 @@
</title>
</svelte:head>
<audio id="audioElement" src="" style="display: none;" />
{#if $showCallOverlay}
<CallOverlay
{submitPrompt}
{stopResponse}
bind:files
modelId={selectedModelIds?.at(0) ?? null}
chatId={$chatId}
{eventTarget}
/>
{/if}
{#if !chatIdProp || (loaded && chatIdProp)}
<div
class="h-screen max-h-[100dvh] {$showSidebar
......@@ -1266,8 +1376,16 @@
bind:files
bind:prompt
bind:autoScroll
bind:selectedToolIds
bind:webSearchEnabled
bind:atSelectedModel
availableToolIds={selectedModelIds.reduce((a, e, i, arr) => {
const model = $models.find((m) => m.id === e);
if (model?.info?.meta?.toolIds ?? false) {
return [...new Set([...a, ...model.info.meta.toolIds])];
}
return a;
}, [])}
{selectedModels}
{messages}
{submitPrompt}
......
src/lib/components/chat/MessageInput.svelte
View file @ 4ff17acc
<script lang="ts">
import { toast } from 'svelte-sonner';
import { onMount, tick, getContext } from 'svelte';
import { type Model, mobile, settings, showSidebar, models, config } from '$lib/stores';
import {
type Model,
mobile,
settings,
showSidebar,
models,
config,
showCallOverlay,
tools,
user as _user
} from '$lib/stores';
import { blobToFile, calculateSHA256, findWordIndices } from '$lib/utils';
import {
......@@ -11,8 +21,6 @@
} from '$lib/apis/rag';
import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS, WEBUI_BASE_URL } from '$lib/constants';
import { transcribeAudio } from '$lib/apis/audio';
import Prompts from './MessageInput/PromptCommands.svelte';
import Suggestions from './MessageInput/Suggestions.svelte';
import AddFilesPlaceholder from '../AddFilesPlaceholder.svelte';
......@@ -21,7 +29,9 @@
import Tooltip from '../common/Tooltip.svelte';
import XMark from '$lib/components/icons/XMark.svelte';
import InputMenu from './MessageInput/InputMenu.svelte';
import { t } from 'i18next';
import Headphone from '../icons/Headphone.svelte';
import VoiceRecording from './MessageInput/VoiceRecording.svelte';
import { transcribeAudio } from '$lib/apis/audio';
const i18n = getContext('i18n');
......@@ -33,6 +43,8 @@
export let atSelectedModel: Model | undefined;
export let selectedModels: [''];
let recording = false;
let chatTextAreaElement: HTMLTextAreaElement;
let filesInputElement;
......@@ -48,14 +60,13 @@
export let files = [];
export let speechRecognitionEnabled = true;
export let availableToolIds = [];
export let selectedToolIds = [];
export let webSearchEnabled = false;
export let prompt = '';
export let messages = [];
let speechRecognition;
let visionCapableModels = [];
$: visionCapableModels = [...(atSelectedModel ? [atSelectedModel] : selectedModels)].filter(
(model) => $models.find((m) => m.id === model)?.info?.meta?.capabilities?.vision ?? true
......@@ -68,176 +79,11 @@
}
}
let mediaRecorder;
let audioChunks = [];
let isRecording = false;
const MIN_DECIBELS = -45;
const scrollToBottom = () => {
const element = document.getElementById('messages-container');
element.scrollTop = element.scrollHeight;
};
const startRecording = async () => {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
mediaRecorder = new MediaRecorder(stream);
mediaRecorder.onstart = () => {
isRecording = true;
console.log('Recording started');
};
mediaRecorder.ondataavailable = (event) => audioChunks.push(event.data);
mediaRecorder.onstop = async () => {
isRecording = false;
console.log('Recording stopped');
// Create a blob from the audio chunks
const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
const file = blobToFile(audioBlob, 'recording.wav');
const res = await transcribeAudio(localStorage.token, file).catch((error) => {
toast.error(error);
return null;
});
if (res) {
prompt = res.text;
await tick();
chatTextAreaElement?.focus();
if (prompt !== '' && $settings?.speechAutoSend === true) {
submitPrompt(prompt, user);
}
}
// saveRecording(audioBlob);
audioChunks = [];
};
// Start recording
mediaRecorder.start();
// Monitor silence
monitorSilence(stream);
};
const monitorSilence = (stream) => {
const audioContext = new AudioContext();
const audioStreamSource = audioContext.createMediaStreamSource(stream);
const analyser = audioContext.createAnalyser();
analyser.minDecibels = MIN_DECIBELS;
audioStreamSource.connect(analyser);
const bufferLength = analyser.frequencyBinCount;
const domainData = new Uint8Array(bufferLength);
let lastSoundTime = Date.now();
const detectSound = () => {
analyser.getByteFrequencyData(domainData);
if (domainData.some((value) => value > 0)) {
lastSoundTime = Date.now();
}
if (isRecording && Date.now() - lastSoundTime > 3000) {
mediaRecorder.stop();
audioContext.close();
return;
}
window.requestAnimationFrame(detectSound);
};
window.requestAnimationFrame(detectSound);
};
const saveRecording = (blob) => {
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
document.body.appendChild(a);
a.style = 'display: none';
a.href = url;
a.download = 'recording.wav';
a.click();
window.URL.revokeObjectURL(url);
};
const speechRecognitionHandler = () => {
// Check if SpeechRecognition is supported
if (isRecording) {
if (speechRecognition) {
speechRecognition.stop();
}
if (mediaRecorder) {
mediaRecorder.stop();
}
} else {
isRecording = true;
if ($settings?.audio?.STTEngine ?? '' !== '') {
startRecording();
} else {
if ('SpeechRecognition' in window || 'webkitSpeechRecognition' in window) {
// Create a SpeechRecognition object
speechRecognition = new (window.SpeechRecognition || window.webkitSpeechRecognition)();
// Set continuous to true for continuous recognition
speechRecognition.continuous = true;
// Set the timeout for turning off the recognition after inactivity (in milliseconds)
const inactivityTimeout = 3000; // 3 seconds
let timeoutId;
// Start recognition
speechRecognition.start();
// Event triggered when speech is recognized
speechRecognition.onresult = async (event) => {
// Clear the inactivity timeout
clearTimeout(timeoutId);
// Handle recognized speech
console.log(event);
const transcript = event.results[Object.keys(event.results).length - 1][0].transcript;
prompt = `${prompt}${transcript}`;
await tick();
chatTextAreaElement?.focus();
// Restart the inactivity timeout
timeoutId = setTimeout(() => {
console.log('Speech recognition turned off due to inactivity.');
speechRecognition.stop();
}, inactivityTimeout);
};
// Event triggered when recognition is ended
speechRecognition.onend = function () {
// Restart recognition after it ends
console.log('recognition ended');
isRecording = false;
if (prompt !== '' && $settings?.speechAutoSend === true) {
submitPrompt(prompt, user);
}
};
// Event triggered when an error occurs
speechRecognition.onerror = function (event) {
console.log(event);
toast.error($i18n.t(`Speech recognition error: {{error}}`, { error: event.error }));
isRecording = false;
};
} else {
toast.error($i18n.t('SpeechRecognition API is not supported in this browser.'));
}
}
}
};
const uploadDoc = async (file) => {
console.log(file);
......@@ -472,7 +318,7 @@
<div class="w-full relative">
{#if prompt.charAt(0) === '/'}
<Prompts bind:this={promptsElement} bind:prompt />
<Prompts bind:this={promptsElement} bind:prompt bind:files />
{:else if prompt.charAt(0) === '#'}
<Documents
bind:this={documentsElement}
......@@ -502,7 +348,6 @@
<Models
bind:this={modelsElement}
bind:prompt
bind:user
bind:chatInputPlaceholder
{messages}
on:select={(e) => {
......@@ -601,431 +446,453 @@
}
}}
/>
<form
dir={$settings?.chatDirection ?? 'LTR'}
class=" flex flex-col relative w-full rounded-3xl px-1.5 bg-gray-50 dark:bg-gray-850 dark:text-gray-100"
on:submit|preventDefault={() => {
// check if selectedModels support image input
submitPrompt(prompt, user);
}}
>
{#if files.length > 0}
<div class="mx-2 mt-2 mb-1 flex flex-wrap gap-2">
{#each files as file, fileIdx}
<div class=" relative group">
{#if file.type === 'image'}
<div class="relative">
<img src={file.url} alt="input" class=" h-16 w-16 rounded-xl object-cover" />
{#if atSelectedModel ? visionCapableModels.length === 0 : selectedModels.length !== visionCapableModels.length}
<Tooltip
className=" absolute top-1 left-1"
content={$i18n.t('{{ models }}', {
models: [...(atSelectedModel ? [atSelectedModel] : selectedModels)]
.filter((id) => !visionCapableModels.includes(id))
.join(', ')
})}
{#if recording}
<VoiceRecording
bind:recording
on:cancel={async () => {
recording = false;
await tick();
document.getElementById('chat-textarea')?.focus();
}}
on:confirm={async (e) => {
const response = e.detail;
prompt = `${prompt}${response} `;
recording = false;
await tick();
document.getElementById('chat-textarea')?.focus();
if ($settings?.speechAutoSend ?? false) {
submitPrompt(prompt);
}
}}
/>
{:else}
<form
class="w-full flex gap-1.5"
on:submit|preventDefault={() => {
// check if selectedModels support image input
submitPrompt(prompt);
}}
>
<div
class="flex-1 flex flex-col relative w-full rounded-3xl px-1.5 bg-gray-50 dark:bg-gray-850 dark:text-gray-100"
dir={$settings?.chatDirection ?? 'LTR'}
>
{#if files.length > 0}
<div class="mx-2 mt-2 mb-1 flex flex-wrap gap-2">
{#each files as file, fileIdx}
<div class=" relative group">
{#if file.type === 'image'}
<div class="relative">
<img
src={file.url}
alt="input"
class=" h-16 w-16 rounded-xl object-cover"
/>
{#if atSelectedModel ? visionCapableModels.length === 0 : selectedModels.length !== visionCapableModels.length}
<Tooltip
className=" absolute top-1 left-1"
content={$i18n.t('{{ models }}', {
models: [...(atSelectedModel ? [atSelectedModel] : selectedModels)]
.filter((id) => !visionCapableModels.includes(id))
.join(', ')
})}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-4 fill-yellow-300"
>
<path
fill-rule="evenodd"
d="M9.401 3.003c1.155-2 4.043-2 5.197 0l7.355 12.748c1.154 2-.29 4.5-2.599 4.5H4.645c-2.309 0-3.752-2.5-2.598-4.5L9.4 3.003ZM12 8.25a.75.75 0 0 1 .75.75v3.75a.75.75 0 0 1-1.5 0V9a.75.75 0 0 1 .75-.75Zm0 8.25a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Z"
clip-rule="evenodd"
/>
</svg>
</Tooltip>
{/if}
</div>
{:else if file.type === 'doc'}
<div
class="h-16 w-[15rem] flex items-center space-x-3 px-2.5 dark:bg-gray-600 rounded-xl border border-gray-200 dark:border-none"
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-4 fill-yellow-300"
>
<path
fill-rule="evenodd"
d="M9.401 3.003c1.155-2 4.043-2 5.197 0l7.355 12.748c1.154 2-.29 4.5-2.599 4.5H4.645c-2.309 0-3.752-2.5-2.598-4.5L9.4 3.003ZM12 8.25a.75.75 0 0 1 .75.75v3.75a.75.75 0 0 1-1.5 0V9a.75.75 0 0 1 .75-.75Zm0 8.25a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Z"
clip-rule="evenodd"
/>
</svg>
</Tooltip>
<div class="p-2.5 bg-red-400 text-white rounded-lg">
{#if file.upload_status}
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="w-6 h-6"
>
<path
fill-rule="evenodd"
d="M5.625 1.5c-1.036 0-1.875.84-1.875 1.875v17.25c0 1.035.84 1.875 1.875 1.875h12.75c1.035 0 1.875-.84 1.875-1.875V12.75A3.75 3.75 0 0 0 16.5 9h-1.875a1.875 1.875 0 0 1-1.875-1.875V5.25A3.75 3.75 0 0 0 9 1.5H5.625ZM7.5 15a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 7.5 15Zm.75 2.25a.75.75 0 0 0 0 1.5H12a.75.75 0 0 0 0-1.5H8.25Z"
clip-rule="evenodd"
/>
<path
d="M12.971 1.816A5.23 5.23 0 0 1 14.25 5.25v1.875c0 .207.168.375.375.375H16.5a5.23 5.23 0 0 1 3.434 1.279 9.768 9.768 0 0 0-6.963-6.963Z"
/>
</svg>
{:else}
<svg
class=" w-6 h-6 translate-y-[0.5px]"
fill="currentColor"
viewBox="0 0 24 24"
xmlns="http://www.w3.org/2000/svg"
><style>
.spinner_qM83 {
animation: spinner_8HQG 1.05s infinite;
}
.spinner_oXPr {
animation-delay: 0.1s;
}
.spinner_ZTLf {
animation-delay: 0.2s;
}
@keyframes spinner_8HQG {
0%,
57.14% {
animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
transform: translate(0);
}
28.57% {
animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
transform: translateY(-6px);
}
100% {
transform: translate(0);
}
}
</style><circle
class="spinner_qM83"
cx="4"
cy="12"
r="2.5"
/><circle
class="spinner_qM83 spinner_oXPr"
cx="12"
cy="12"
r="2.5"
/><circle
class="spinner_qM83 spinner_ZTLf"
cx="20"
cy="12"
r="2.5"
/></svg
>
{/if}
</div>
<div class="flex flex-col justify-center -space-y-0.5">
<div class=" dark:text-gray-100 text-sm font-medium line-clamp-1">
{file.name}
</div>
<div class=" text-gray-500 text-sm">{$i18n.t('Document')}</div>
</div>
</div>
{:else if file.type === 'collection'}
<div
class="h-16 w-[15rem] flex items-center space-x-3 px-2.5 dark:bg-gray-600 rounded-xl border border-gray-200 dark:border-none"
>
<div class="p-2.5 bg-red-400 text-white rounded-lg">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="w-6 h-6"
>
<path
d="M7.5 3.375c0-1.036.84-1.875 1.875-1.875h.375a3.75 3.75 0 0 1 3.75 3.75v1.875C13.5 8.161 14.34 9 15.375 9h1.875A3.75 3.75 0 0 1 21 12.75v3.375C21 17.16 20.16 18 19.125 18h-9.75A1.875 1.875 0 0 1 7.5 16.125V3.375Z"
/>
<path
d="M15 5.25a5.23 5.23 0 0 0-1.279-3.434 9.768 9.768 0 0 1 6.963 6.963A5.23 5.23 0 0 0 17.25 7.5h-1.875A.375.375 0 0 1 15 7.125V5.25ZM4.875 6H6v10.125A3.375 3.375 0 0 0 9.375 19.5H16.5v1.125c0 1.035-.84 1.875-1.875 1.875h-9.75A1.875 1.875 0 0 1 3 20.625V7.875C3 6.839 3.84 6 4.875 6Z"
/>
</svg>
</div>
<div class="flex flex-col justify-center -space-y-0.5">
<div class=" dark:text-gray-100 text-sm font-medium line-clamp-1">
{file?.title ?? `#${file.name}`}
</div>
<div class=" text-gray-500 text-sm">{$i18n.t('Collection')}</div>
</div>
</div>
{/if}
</div>
{:else if file.type === 'doc'}
<div
class="h-16 w-[15rem] flex items-center space-x-3 px-2.5 dark:bg-gray-600 rounded-xl border border-gray-200 dark:border-none"
>
<div class="p-2.5 bg-red-400 text-white rounded-lg">
{#if file.upload_status}
<div class=" absolute -top-1 -right-1">
<button
class=" bg-gray-400 text-white border border-white rounded-full group-hover:visible invisible transition"
type="button"
on:click={() => {
files.splice(fileIdx, 1);
files = files;
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
viewBox="0 0 20 20"
fill="currentColor"
class="w-6 h-6"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M5.625 1.5c-1.036 0-1.875.84-1.875 1.875v17.25c0 1.035.84 1.875 1.875 1.875h12.75c1.035 0 1.875-.84 1.875-1.875V12.75A3.75 3.75 0 0 0 16.5 9h-1.875a1.875 1.875 0 0 1-1.875-1.875V5.25A3.75 3.75 0 0 0 9 1.5H5.625ZM7.5 15a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 7.5 15Zm.75 2.25a.75.75 0 0 0 0 1.5H12a.75.75 0 0 0 0-1.5H8.25Z"
clip-rule="evenodd"
/>
<path
d="M12.971 1.816A5.23 5.23 0 0 1 14.25 5.25v1.875c0 .207.168.375.375.375H16.5a5.23 5.23 0 0 1 3.434 1.279 9.768 9.768 0 0 0-6.963-6.963Z"
d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
/>
</svg>
{:else}
<svg
class=" w-6 h-6 translate-y-[0.5px]"
fill="currentColor"
viewBox="0 0 24 24"
xmlns="http://www.w3.org/2000/svg"
><style>
.spinner_qM83 {
animation: spinner_8HQG 1.05s infinite;
}
.spinner_oXPr {
animation-delay: 0.1s;
}
.spinner_ZTLf {
animation-delay: 0.2s;
}
@keyframes spinner_8HQG {
0%,
57.14% {
animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
transform: translate(0);
}
28.57% {
animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
transform: translateY(-6px);
}
100% {
transform: translate(0);
}
}
</style><circle class="spinner_qM83" cx="4" cy="12" r="2.5" /><circle
class="spinner_qM83 spinner_oXPr"
cx="12"
cy="12"
r="2.5"
/><circle
class="spinner_qM83 spinner_ZTLf"
cx="20"
cy="12"
r="2.5"
/></svg
>
{/if}
</div>
<div class="flex flex-col justify-center -space-y-0.5">
<div class=" dark:text-gray-100 text-sm font-medium line-clamp-1">
{file.name}
</div>
<div class=" text-gray-500 text-sm">{$i18n.t('Document')}</div>
</div>
</div>
{:else if file.type === 'collection'}
<div
class="h-16 w-[15rem] flex items-center space-x-3 px-2.5 dark:bg-gray-600 rounded-xl border border-gray-200 dark:border-none"
>
<div class="p-2.5 bg-red-400 text-white rounded-lg">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="w-6 h-6"
>
<path
d="M7.5 3.375c0-1.036.84-1.875 1.875-1.875h.375a3.75 3.75 0 0 1 3.75 3.75v1.875C13.5 8.161 14.34 9 15.375 9h1.875A3.75 3.75 0 0 1 21 12.75v3.375C21 17.16 20.16 18 19.125 18h-9.75A1.875 1.875 0 0 1 7.5 16.125V3.375Z"
/>
<path
d="M15 5.25a5.23 5.23 0 0 0-1.279-3.434 9.768 9.768 0 0 1 6.963 6.963A5.23 5.23 0 0 0 17.25 7.5h-1.875A.375.375 0 0 1 15 7.125V5.25ZM4.875 6H6v10.125A3.375 3.375 0 0 0 9.375 19.5H16.5v1.125c0 1.035-.84 1.875-1.875 1.875h-9.75A1.875 1.875 0 0 1 3 20.625V7.875C3 6.839 3.84 6 4.875 6Z"
/>
</svg>
</div>
<div class="flex flex-col justify-center -space-y-0.5">
<div class=" dark:text-gray-100 text-sm font-medium line-clamp-1">
{file?.title ?? `#${file.name}`}
</div>
<div class=" text-gray-500 text-sm">{$i18n.t('Collection')}</div>
</button>
</div>
</div>
{/if}
{/each}
</div>
{/if}
<div class=" absolute -top-1 -right-1">
<div class=" flex">
<div class=" ml-0.5 self-end mb-1.5 flex space-x-1">
<InputMenu
bind:webSearchEnabled
bind:selectedToolIds
tools={$tools.reduce((a, e, i, arr) => {
if (availableToolIds.includes(e.id) || ($_user?.role ?? 'user') === 'admin') {
a[e.id] = {
name: e.name,
description: e.meta.description,
enabled: false
};
}
return a;
}, {})}
uploadFilesHandler={() => {
filesInputElement.click();
}}
onClose={async () => {
await tick();
chatTextAreaElement?.focus();
}}
>
<button
class=" bg-gray-400 text-white border border-white rounded-full group-hover:visible invisible transition"
class="bg-gray-50 hover:bg-gray-100 text-gray-800 dark:bg-gray-850 dark:text-white dark:hover:bg-gray-800 transition rounded-full p-2 outline-none focus:outline-none"
type="button"
on:click={() => {
files.splice(fileIdx, 1);
files = files;
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
class="size-5"
>
<path
d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
/>
</svg>
</button>
</div>
</InputMenu>
</div>
{/each}
</div>
{/if}
<div class=" flex">
<div class=" ml-1 self-end mb-2 flex space-x-1">
<InputMenu
bind:webSearchEnabled
uploadFilesHandler={() => {
filesInputElement.click();
}}
onClose={async () => {
await tick();
chatTextAreaElement?.focus();
}}
>
<button
class="bg-gray-50 hover:bg-gray-100 text-gray-800 dark:bg-gray-850 dark:text-white dark:hover:bg-gray-800 transition rounded-full p-1.5 outline-none focus:outline-none"
type="button"
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="size-5"
>
<path
d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
/>
</svg>
</button>
</InputMenu>
</div>
<textarea
id="chat-textarea"
bind:this={chatTextAreaElement}
class="scrollbar-hidden bg-gray-50 dark:bg-gray-850 dark:text-gray-100 outline-none w-full py-3 px-3 rounded-xl resize-none h-[48px]"
placeholder={chatInputPlaceholder !== ''
? chatInputPlaceholder
: isRecording
? $i18n.t('Listening...')
: $i18n.t('Send a Message')}
bind:value={prompt}
on:keypress={(e) => {
if (
!$mobile ||
!(
'ontouchstart' in window ||
navigator.maxTouchPoints > 0 ||
navigator.msMaxTouchPoints > 0
)
) {
// Prevent Enter key from creating a new line
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault();
}
<textarea
id="chat-textarea"
bind:this={chatTextAreaElement}
class="scrollbar-hidden bg-gray-50 dark:bg-gray-850 dark:text-gray-100 outline-none w-full py-3 px-1 rounded-xl resize-none h-[48px]"
placeholder={chatInputPlaceholder !== ''
? chatInputPlaceholder
: $i18n.t('Send a Message')}
bind:value={prompt}
on:keypress={(e) => {
if (
!$mobile ||
!(
'ontouchstart' in window ||
navigator.maxTouchPoints > 0 ||
navigator.msMaxTouchPoints > 0
)
) {
// Prevent Enter key from creating a new line
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault();
}
// Submit the prompt when Enter key is pressed
if (prompt !== '' && e.key === 'Enter' && !e.shiftKey) {
submitPrompt(prompt, user);
}
}
}}
on:keydown={async (e) => {
const isCtrlPressed = e.ctrlKey || e.metaKey; // metaKey is for Cmd key on Mac
// Submit the prompt when Enter key is pressed
if (prompt !== '' && e.key === 'Enter' && !e.shiftKey) {
submitPrompt(prompt);
}
}
}}
on:keydown={async (e) => {
const isCtrlPressed = e.ctrlKey || e.metaKey; // metaKey is for Cmd key on Mac
// Check if Ctrl + R is pressed
if (prompt === '' && isCtrlPressed && e.key.toLowerCase() === 'r') {
e.preventDefault();
console.log('regenerate');
// Check if Ctrl + R is pressed
if (prompt === '' && isCtrlPressed && e.key.toLowerCase() === 'r') {
e.preventDefault();
console.log('regenerate');
const regenerateButton = [
...document.getElementsByClassName('regenerate-response-button')
]?.at(-1);
const regenerateButton = [
...document.getElementsByClassName('regenerate-response-button')
]?.at(-1);
regenerateButton?.click();
}
regenerateButton?.click();
}
if (prompt === '' && e.key == 'ArrowUp') {
e.preventDefault();
if (prompt === '' && e.key == 'ArrowUp') {
e.preventDefault();
const userMessageElement = [
...document.getElementsByClassName('user-message')
]?.at(-1);
const userMessageElement = [
...document.getElementsByClassName('user-message')
]?.at(-1);
const editButton = [
...document.getElementsByClassName('edit-user-message-button')
]?.at(-1);
const editButton = [
...document.getElementsByClassName('edit-user-message-button')
]?.at(-1);
console.log(userMessageElement);
console.log(userMessageElement);
userMessageElement.scrollIntoView({ block: 'center' });
editButton?.click();
}
userMessageElement.scrollIntoView({ block: 'center' });
editButton?.click();
}
if (['/', '#', '@'].includes(prompt.charAt(0)) && e.key === 'ArrowUp') {
e.preventDefault();
if (['/', '#', '@'].includes(prompt.charAt(0)) && e.key === 'ArrowUp') {
e.preventDefault();
(promptsElement || documentsElement || modelsElement).selectUp();
(promptsElement || documentsElement || modelsElement).selectUp();
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
]?.at(-1);
commandOptionButton.scrollIntoView({ block: 'center' });
}
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
]?.at(-1);
commandOptionButton.scrollIntoView({ block: 'center' });
}
if (['/', '#', '@'].includes(prompt.charAt(0)) && e.key === 'ArrowDown') {
e.preventDefault();
if (['/', '#', '@'].includes(prompt.charAt(0)) && e.key === 'ArrowDown') {
e.preventDefault();
(promptsElement || documentsElement || modelsElement).selectDown();
(promptsElement || documentsElement || modelsElement).selectDown();
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
]?.at(-1);
commandOptionButton.scrollIntoView({ block: 'center' });
}
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
]?.at(-1);
commandOptionButton.scrollIntoView({ block: 'center' });
}
if (['/', '#', '@'].includes(prompt.charAt(0)) && e.key === 'Enter') {
e.preventDefault();
if (['/', '#', '@'].includes(prompt.charAt(0)) && e.key === 'Enter') {
e.preventDefault();
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
]?.at(-1);
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
]?.at(-1);
if (e.shiftKey) {
prompt = `${prompt}\n`;
} else if (commandOptionButton) {
commandOptionButton?.click();
} else {
document.getElementById('send-message-button')?.click();
}
}
if (e.shiftKey) {
prompt = `${prompt}\n`;
} else if (commandOptionButton) {
commandOptionButton?.click();
} else {
document.getElementById('send-message-button')?.click();
}
}
if (['/', '#', '@'].includes(prompt.charAt(0)) && e.key === 'Tab') {
e.preventDefault();
if (['/', '#', '@'].includes(prompt.charAt(0)) && e.key === 'Tab') {
e.preventDefault();
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
]?.at(-1);
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
]?.at(-1);
commandOptionButton?.click();
} else if (e.key === 'Tab') {
const words = findWordIndices(prompt);
commandOptionButton?.click();
} else if (e.key === 'Tab') {
const words = findWordIndices(prompt);
if (words.length > 0) {
const word = words.at(0);
const fullPrompt = prompt;
if (words.length > 0) {
const word = words.at(0);
const fullPrompt = prompt;
prompt = prompt.substring(0, word?.endIndex + 1);
await tick();
prompt = prompt.substring(0, word?.endIndex + 1);
await tick();
e.target.scrollTop = e.target.scrollHeight;
prompt = fullPrompt;
await tick();
e.target.scrollTop = e.target.scrollHeight;
prompt = fullPrompt;
await tick();
e.preventDefault();
e.target.setSelectionRange(word?.startIndex, word.endIndex + 1);
}
e.preventDefault();
e.target.setSelectionRange(word?.startIndex, word.endIndex + 1);
}
e.target.style.height = '';
e.target.style.height = Math.min(e.target.scrollHeight, 200) + 'px';
}
e.target.style.height = '';
e.target.style.height = Math.min(e.target.scrollHeight, 200) + 'px';
}
if (e.key === 'Escape') {
console.log('Escape');
atSelectedModel = undefined;
}
}}
rows="1"
on:input={(e) => {
e.target.style.height = '';
e.target.style.height = Math.min(e.target.scrollHeight, 200) + 'px';
user = null;
}}
on:focus={(e) => {
e.target.style.height = '';
e.target.style.height = Math.min(e.target.scrollHeight, 200) + 'px';
}}
on:paste={(e) => {
const clipboardData = e.clipboardData || window.clipboardData;
if (clipboardData && clipboardData.items) {
for (const item of clipboardData.items) {
if (item.type.indexOf('image') !== -1) {
const blob = item.getAsFile();
const reader = new FileReader();
reader.onload = function (e) {
files = [
...files,
{
type: 'image',
url: `${e.target.result}`
}
];
};
if (e.key === 'Escape') {
console.log('Escape');
atSelectedModel = undefined;
}
}}
rows="1"
on:input={(e) => {
e.target.style.height = '';
e.target.style.height = Math.min(e.target.scrollHeight, 200) + 'px';
user = null;
}}
on:focus={(e) => {
e.target.style.height = '';
e.target.style.height = Math.min(e.target.scrollHeight, 200) + 'px';
}}
on:paste={(e) => {
const clipboardData = e.clipboardData || window.clipboardData;
if (clipboardData && clipboardData.items) {
for (const item of clipboardData.items) {
if (item.type.indexOf('image') !== -1) {
const blob = item.getAsFile();
const reader = new FileReader();
reader.onload = function (e) {
files = [
...files,
{
type: 'image',
url: `${e.target.result}`
}
];
};
reader.readAsDataURL(blob);
reader.readAsDataURL(blob);
}
}
}
}
}
}}
/>
}}
/>
<div class="self-end mb-2 flex space-x-1 mr-1">
{#if messages.length == 0 || messages.at(-1).done == true}
<Tooltip content={$i18n.t('Record voice')}>
{#if speechRecognitionEnabled}
<button
id="voice-input-button"
class=" text-gray-600 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-850 transition rounded-full p-1.5 mr-0.5 self-center"
type="button"
on:click={() => {
speechRecognitionHandler();
}}
>
{#if isRecording}
<svg
class=" w-5 h-5 translate-y-[0.5px]"
fill="currentColor"
viewBox="0 0 24 24"
xmlns="http://www.w3.org/2000/svg"
><style>
.spinner_qM83 {
animation: spinner_8HQG 1.05s infinite;
}
.spinner_oXPr {
animation-delay: 0.1s;
<div class="self-end mb-2 flex space-x-1 mr-1">
{#if messages.length == 0 || messages.at(-1).done == true}
<Tooltip content={$i18n.t('Record voice')}>
<button
id="voice-input-button"
class=" text-gray-600 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-850 transition rounded-full p-1.5 mr-0.5 self-center"
type="button"
on:click={async () => {
try {
const res = await navigator.mediaDevices
.getUserMedia({ audio: true })
.catch(function (err) {
toast.error(
$i18n.t(
`Permission denied when accessing microphone: {{error}}`,
{
error: err
}
)
);
return null;
});
if (res) {
recording = true;
}
.spinner_ZTLf {
animation-delay: 0.2s;
}
@keyframes spinner_8HQG {
0%,
57.14% {
animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
transform: translate(0);
}
28.57% {
animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
transform: translateY(-6px);
}
100% {
transform: translate(0);
}
}
</style><circle class="spinner_qM83" cx="4" cy="12" r="2.5" /><circle
class="spinner_qM83 spinner_oXPr"
cx="12"
cy="12"
r="2.5"
/><circle
class="spinner_qM83 spinner_ZTLf"
cx="20"
cy="12"
r="2.5"
/></svg
>
{:else}
} catch {
toast.error($i18n.t('Permission denied when accessing microphone'));
}
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
......@@ -1037,58 +904,105 @@
d="M5.5 9.643a.75.75 0 00-1.5 0V10c0 3.06 2.29 5.585 5.25 5.954V17.5h-1.5a.75.75 0 000 1.5h4.5a.75.75 0 000-1.5h-1.5v-1.546A6.001 6.001 0 0016 10v-.357a.75.75 0 00-1.5 0V10a4.5 4.5 0 01-9 0v-.357z"
/>
</svg>
{/if}
</button>
</button>
</Tooltip>
{/if}
</Tooltip>
</div>
</div>
</div>
<div class="flex items-end w-10">
{#if messages.length == 0 || messages.at(-1).done == true}
{#if prompt === ''}
<div class=" flex items-center mb-1">
<Tooltip content={$i18n.t('Call')}>
<button
class=" text-gray-600 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-850 transition rounded-full p-2 self-center"
type="button"
on:click={async () => {
if (selectedModels.length > 1) {
toast.error($i18n.t('Select only one model to call'));
return;
}
if ($config.audio.stt.engine === 'web') {
toast.error(
$i18n.t('Call feature is not supported when using Web STT engine')
);
<Tooltip content={$i18n.t('Send message')}>
return;
}
// check if user has access to getUserMedia
try {
await navigator.mediaDevices.getUserMedia({ audio: true });
// If the user grants the permission, proceed to show the call overlay
showCallOverlay.set(true);
} catch (err) {
// If the user denies the permission or an error occurs, show an error message
toast.error($i18n.t('Permission denied when accessing media devices'));
}
}}
>
<Headphone className="size-6" />
</button>
</Tooltip>
</div>
{:else}
<div class=" flex items-center mb-1">
<Tooltip content={$i18n.t('Send message')}>
<button
id="send-message-button"
class="{prompt !== ''
? 'bg-black text-white hover:bg-gray-900 dark:bg-white dark:text-black dark:hover:bg-gray-100 '
: 'text-white bg-gray-200 dark:text-gray-900 dark:bg-gray-700 disabled'} transition rounded-full p-1.5 m-0.5 self-center"
type="submit"
disabled={prompt === ''}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="size-6"
>
<path
fill-rule="evenodd"
d="M8 14a.75.75 0 0 1-.75-.75V4.56L4.03 7.78a.75.75 0 0 1-1.06-1.06l4.5-4.5a.75.75 0 0 1 1.06 0l4.5 4.5a.75.75 0 0 1-1.06 1.06L8.75 4.56v8.69A.75.75 0 0 1 8 14Z"
clip-rule="evenodd"
/>
</svg>
</button>
</Tooltip>
</div>
{/if}
{:else}
<div class=" flex items-center mb-1.5">
<button
id="send-message-button"
class="{prompt !== ''
? 'bg-black text-white hover:bg-gray-900 dark:bg-white dark:text-black dark:hover:bg-gray-100 '
: 'text-white bg-gray-200 dark:text-gray-900 dark:bg-gray-700 disabled'} transition rounded-full p-1.5 self-center"
type="submit"
disabled={prompt === ''}
class="bg-white hover:bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-white dark:hover:bg-gray-800 transition rounded-full p-1.5"
on:click={() => {
stopResponse();
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
viewBox="0 0 24 24"
fill="currentColor"
class="w-5 h-5"
class="size-6"
>
<path
fill-rule="evenodd"
d="M8 14a.75.75 0 0 1-.75-.75V4.56L4.03 7.78a.75.75 0 0 1-1.06-1.06l4.5-4.5a.75.75 0 0 1 1.06 0l4.5 4.5a.75.75 0 0 1-1.06 1.06L8.75 4.56v8.69A.75.75 0 0 1 8 14Z"
d="M2.25 12c0-5.385 4.365-9.75 9.75-9.75s9.75 4.365 9.75 9.75-4.365 9.75-9.75 9.75S2.25 17.385 2.25 12zm6-2.438c0-.724.588-1.312 1.313-1.312h4.874c.725 0 1.313.588 1.313 1.313v4.874c0 .725-.588 1.313-1.313 1.313H9.564a1.312 1.312 0 01-1.313-1.313V9.564z"
clip-rule="evenodd"
/>
</svg>
</button>
</Tooltip>
{:else}
<button
class="bg-white hover:bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-white dark:hover:bg-gray-800 transition rounded-full p-1.5"
on:click={stopResponse}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="w-5 h-5"
>
<path
fill-rule="evenodd"
d="M2.25 12c0-5.385 4.365-9.75 9.75-9.75s9.75 4.365 9.75 9.75-4.365 9.75-9.75 9.75S2.25 17.385 2.25 12zm6-2.438c0-.724.588-1.312 1.313-1.312h4.874c.725 0 1.313.588 1.313 1.313v4.874c0 .725-.588 1.313-1.313 1.313H9.564a1.312 1.312 0 01-1.313-1.313V9.564z"
clip-rule="evenodd"
/>
</svg>
</button>
</div>
{/if}
</div>
</div>
</form>
</form>
{/if}
<div class="mt-1.5 text-xs text-gray-500 text-center">
<div class="mt-1.5 text-xs text-gray-500 text-center line-clamp-1">
{$i18n.t('LLMs can make mistakes. Verify important information.')}
</div>
</div>
......
src/lib/components/chat/MessageInput/CallOverlay.svelte 0 → 100644
View file @ 4ff17acc
<script lang="ts">
import { config, settings, showCallOverlay } from '$lib/stores';
import { onMount, tick, getContext } from 'svelte';
import {
blobToFile,
calculateSHA256,
extractSentencesForAudio,
findWordIndices
} from '$lib/utils';
import { generateEmoji } from '$lib/apis';
import { synthesizeOpenAISpeech, transcribeAudio } from '$lib/apis/audio';
import { toast } from 'svelte-sonner';
import Tooltip from '$lib/components/common/Tooltip.svelte';
import VideoInputMenu from './CallOverlay/VideoInputMenu.svelte';
const i18n = getContext('i18n');
export let eventTarget: EventTarget;
export let submitPrompt: Function;
export let stopResponse: Function;
export let files;
export let chatId;
export let modelId;
let loading = false;
let confirmed = false;
let interrupted = false;
let emoji = null;
let camera = false;
let cameraStream = null;
let chatStreaming = false;
let rmsLevel = 0;
let hasStartedSpeaking = false;
let mediaRecorder;
let audioChunks = [];
let videoInputDevices = [];
let selectedVideoInputDeviceId = null;
const getVideoInputDevices = async () => {
const devices = await navigator.mediaDevices.enumerateDevices();
videoInputDevices = devices.filter((device) => device.kind === 'videoinput');
if (!!navigator.mediaDevices.getDisplayMedia) {
videoInputDevices = [
...videoInputDevices,
{
deviceId: 'screen',
label: 'Screen Share'
}
];
}
console.log(videoInputDevices);
if (selectedVideoInputDeviceId === null && videoInputDevices.length > 0) {
selectedVideoInputDeviceId = videoInputDevices[0].deviceId;
}
};
const startCamera = async () => {
await getVideoInputDevices();
if (cameraStream === null) {
camera = true;
await tick();
try {
await startVideoStream();
} catch (err) {
console.error('Error accessing webcam: ', err);
}
}
};
const startVideoStream = async () => {
const video = document.getElementById('camera-feed');
if (video) {
if (selectedVideoInputDeviceId === 'screen') {
cameraStream = await navigator.mediaDevices.getDisplayMedia({
video: {
cursor: 'always'
},
audio: false
});
} else {
cameraStream = await navigator.mediaDevices.getUserMedia({
video: {
deviceId: selectedVideoInputDeviceId ? { exact: selectedVideoInputDeviceId } : undefined
}
});
}
if (cameraStream) {
await getVideoInputDevices();
video.srcObject = cameraStream;
await video.play();
}
}
};
const stopVideoStream = async () => {
if (cameraStream) {
const tracks = cameraStream.getTracks();
tracks.forEach((track) => track.stop());
}
cameraStream = null;
};
const takeScreenshot = () => {
const video = document.getElementById('camera-feed');
const canvas = document.getElementById('camera-canvas');
if (!canvas) {
return;
}
const context = canvas.getContext('2d');
// Make the canvas match the video dimensions
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
// Draw the image from the video onto the canvas
context.drawImage(video, 0, 0, video.videoWidth, video.videoHeight);
// Convert the canvas to a data base64 URL and console log it
const dataURL = canvas.toDataURL('image/png');
console.log(dataURL);
return dataURL;
};
const stopCamera = async () => {
await stopVideoStream();
camera = false;
};
const MIN_DECIBELS = -55;
const VISUALIZER_BUFFER_LENGTH = 300;
const transcribeHandler = async (audioBlob) => {
// Create a blob from the audio chunks
await tick();
const file = blobToFile(audioBlob, 'recording.wav');
const res = await transcribeAudio(localStorage.token, file).catch((error) => {
toast.error(error);
return null;
});
if (res) {
console.log(res.text);
if (res.text !== '') {
const _responses = await submitPrompt(res.text, { _raw: true });
console.log(_responses);
}
}
};
const stopRecordingCallback = async (_continue = true) => {
if ($showCallOverlay) {
console.log('%c%s', 'color: red; font-size: 20px;', '🚨 stopRecordingCallback 🚨');
// deep copy the audioChunks array
const _audioChunks = audioChunks.slice(0);
audioChunks = [];
mediaRecorder = false;
if (_continue) {
startRecording();
}
if (confirmed) {
loading = true;
emoji = null;
if (cameraStream) {
const imageUrl = takeScreenshot();
files = [
{
type: 'image',
url: imageUrl
}
];
}
const audioBlob = new Blob(_audioChunks, { type: 'audio/wav' });
await transcribeHandler(audioBlob);
confirmed = false;
loading = false;
}
} else {
audioChunks = [];
mediaRecorder = false;
}
};
const startRecording = async () => {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
mediaRecorder = new MediaRecorder(stream);
mediaRecorder.onstart = () => {
console.log('Recording started');
audioChunks = [];
analyseAudio(stream);
};
mediaRecorder.ondataavailable = (event) => {
if (hasStartedSpeaking) {
audioChunks.push(event.data);
}
};
mediaRecorder.onstop = (e) => {
console.log('Recording stopped', e);
stopRecordingCallback();
};
mediaRecorder.start();
};
// Function to calculate the RMS level from time domain data
const calculateRMS = (data: Uint8Array) => {
let sumSquares = 0;
for (let i = 0; i < data.length; i++) {
const normalizedValue = (data[i] - 128) / 128; // Normalize the data
sumSquares += normalizedValue * normalizedValue;
}
return Math.sqrt(sumSquares / data.length);
};
const analyseAudio = (stream) => {
const audioContext = new AudioContext();
const audioStreamSource = audioContext.createMediaStreamSource(stream);
const analyser = audioContext.createAnalyser();
analyser.minDecibels = MIN_DECIBELS;
audioStreamSource.connect(analyser);
const bufferLength = analyser.frequencyBinCount;
const domainData = new Uint8Array(bufferLength);
const timeDomainData = new Uint8Array(analyser.fftSize);
let lastSoundTime = Date.now();
hasStartedSpeaking = false;
console.log('🔊 Sound detection started', lastSoundTime, hasStartedSpeaking);
const detectSound = () => {
const processFrame = () => {
if (!mediaRecorder || !$showCallOverlay) {
return;
}
analyser.getByteTimeDomainData(timeDomainData);
analyser.getByteFrequencyData(domainData);
// Calculate RMS level from time domain data
rmsLevel = calculateRMS(timeDomainData);
// Check if initial speech/noise has started
const hasSound = domainData.some((value) => value > 0);
if (hasSound) {
// BIG RED TEXT
console.log('%c%s', 'color: red; font-size: 20px;', '🔊 Sound detected');
if (!hasStartedSpeaking) {
hasStartedSpeaking = true;
stopAllAudio();
}
lastSoundTime = Date.now();
}
// Start silence detection only after initial speech/noise has been detected
if (hasStartedSpeaking) {
if (Date.now() - lastSoundTime > 2000) {
confirmed = true;
if (mediaRecorder) {
console.log('%c%s', 'color: red; font-size: 20px;', '🔇 Silence detected');
mediaRecorder.stop();
return;
}
}
}
window.requestAnimationFrame(processFrame);
};
window.requestAnimationFrame(processFrame);
};
detectSound();
};
let finishedMessages = {};
let currentMessageId = null;
let currentUtterance = null;
const speakSpeechSynthesisHandler = (content) => {
if ($showCallOverlay) {
return new Promise((resolve) => {
let voices = [];
const getVoicesLoop = setInterval(async () => {
voices = await speechSynthesis.getVoices();
if (voices.length > 0) {
clearInterval(getVoicesLoop);
const voice =
voices
?.filter(
(v) => v.voiceURI === ($settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice)
)
?.at(0) ?? undefined;
currentUtterance = new SpeechSynthesisUtterance(content);
if (voice) {
currentUtterance.voice = voice;
}
speechSynthesis.speak(currentUtterance);
currentUtterance.onend = async (e) => {
await new Promise((r) => setTimeout(r, 200));
resolve(e);
};
}
}, 100);
});
} else {
return Promise.resolve();
}
};
const playAudio = (audio) => {
if ($showCallOverlay) {
return new Promise((resolve) => {
const audioElement = document.getElementById('audioElement');
if (audioElement) {
audioElement.src = audio.src;
audioElement.muted = true;
audioElement
.play()
.then(() => {
audioElement.muted = false;
})
.catch((error) => {
console.error(error);
});
audioElement.onended = async (e) => {
await new Promise((r) => setTimeout(r, 100));
resolve(e);
};
}
});
} else {
return Promise.resolve();
}
};
const stopAllAudio = async () => {
interrupted = true;
if (chatStreaming) {
stopResponse();
}
if (currentUtterance) {
speechSynthesis.cancel();
currentUtterance = null;
}
const audioElement = document.getElementById('audioElement');
if (audioElement) {
audioElement.muted = true;
audioElement.pause();
audioElement.currentTime = 0;
}
};
let audioAbortController = new AbortController();
// Audio cache map where key is the content and value is the Audio object.
const audioCache = new Map();
const emojiCache = new Map();
const fetchAudio = async (content) => {
if (!audioCache.has(content)) {
try {
// Set the emoji for the content if needed
if ($settings?.showEmojiInCall ?? false) {
const emoji = await generateEmoji(localStorage.token, modelId, content, chatId);
if (emoji) {
emojiCache.set(content, emoji);
}
}
if ($config.audio.tts.engine !== '') {
const res = await synthesizeOpenAISpeech(
localStorage.token,
$settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice,
content
).catch((error) => {
console.error(error);
return null;
});
if (res) {
const blob = await res.blob();
const blobUrl = URL.createObjectURL(blob);
audioCache.set(content, new Audio(blobUrl));
}
} else {
audioCache.set(content, true);
}
} catch (error) {
console.error('Error synthesizing speech:', error);
}
}
return audioCache.get(content);
};
let messages = {};
const monitorAndPlayAudio = async (id, signal) => {
while (!signal.aborted) {
if (messages[id] && messages[id].length > 0) {
// Retrieve the next content string from the queue
const content = messages[id].shift(); // Dequeues the content for playing
if (audioCache.has(content)) {
// If content is available in the cache, play it
// Set the emoji for the content if available
if (($settings?.showEmojiInCall ?? false) && emojiCache.has(content)) {
emoji = emojiCache.get(content);
} else {
emoji = null;
}
if ($config.audio.tts.engine !== '') {
try {
console.log(
'%c%s',
'color: red; font-size: 20px;',
`Playing audio for content: ${content}`
);
const audio = audioCache.get(content);
await playAudio(audio); // Here ensure that playAudio is indeed correct method to execute
console.log(`Played audio for content: ${content}`);
await new Promise((resolve) => setTimeout(resolve, 200)); // Wait before retrying to reduce tight loop
} catch (error) {
console.error('Error playing audio:', error);
}
} else {
await speakSpeechSynthesisHandler(content);
}
} else {
// If not available in the cache, push it back to the queue and delay
messages[id].unshift(content); // Re-queue the content at the start
console.log(`Audio for "${content}" not yet available in the cache, re-queued...`);
await new Promise((resolve) => setTimeout(resolve, 200)); // Wait before retrying to reduce tight loop
}
} else if (finishedMessages[id] && messages[id] && messages[id].length === 0) {
// If the message is finished and there are no more messages to process, break the loop
break;
} else {
// No messages to process, sleep for a bit
await new Promise((resolve) => setTimeout(resolve, 200));
}
}
console.log(`Audio monitoring and playing stopped for message ID ${id}`);
};
onMount(async () => {
startRecording();
const chatStartHandler = async (e) => {
const { id } = e.detail;
chatStreaming = true;
if (currentMessageId !== id) {
console.log(`Received chat start event for message ID ${id}`);
currentMessageId = id;
if (audioAbortController) {
audioAbortController.abort();
}
audioAbortController = new AbortController();
// Start monitoring and playing audio for the message ID
monitorAndPlayAudio(id, audioAbortController.signal);
}
};
const chatEventHandler = async (e) => {
const { id, content } = e.detail;
// "id" here is message id
// if "id" is not the same as "currentMessageId" then do not process
// "content" here is a sentence from the assistant,
// there will be many sentences for the same "id"
if (currentMessageId === id) {
console.log(`Received chat event for message ID ${id}: ${content}`);
try {
if (messages[id] === undefined) {
messages[id] = [content];
} else {
messages[id].push(content);
}
console.log(content);
fetchAudio(content);
} catch (error) {
console.error('Failed to fetch or play audio:', error);
}
}
};
const chatFinishHandler = async (e) => {
const { id, content } = e.detail;
// "content" here is the entire message from the assistant
chatStreaming = false;
finishedMessages[id] = true;
};
eventTarget.addEventListener('chat:start', chatStartHandler);
eventTarget.addEventListener('chat', chatEventHandler);
eventTarget.addEventListener('chat:finish', chatFinishHandler);
return async () => {
eventTarget.removeEventListener('chat:start', chatStartHandler);
eventTarget.removeEventListener('chat', chatEventHandler);
eventTarget.removeEventListener('chat:finish', chatFinishHandler);
audioAbortController.abort();
await tick();
await stopAllAudio();
await stopRecordingCallback(false);
await stopCamera();
};
});
</script>
{#if $showCallOverlay}
<div class=" absolute w-full h-screen max-h-[100dvh] flex z-[999] overflow-hidden">
<div
class="absolute w-full h-screen max-h-[100dvh] bg-white text-gray-700 dark:bg-black dark:text-gray-300 flex justify-center"
>
<div class="max-w-lg w-full h-screen max-h-[100dvh] flex flex-col justify-between p-3 md:p-6">
{#if camera}
<div class="flex justify-center items-center w-full h-20 min-h-20">
{#if emoji}
<div
class=" transition-all rounded-full"
style="font-size:{rmsLevel * 100 > 4
? '4.5'
: rmsLevel * 100 > 2
? '4.25'
: rmsLevel * 100 > 1
? '3.75'
: '3.5'}rem;width: 100%; text-align:center;"
>
{emoji}
</div>
{:else if loading}
<svg
class="size-12 text-gray-900 dark:text-gray-400"
viewBox="0 0 24 24"
fill="currentColor"
xmlns="http://www.w3.org/2000/svg"
><style>
.spinner_qM83 {
animation: spinner_8HQG 1.05s infinite;
}
.spinner_oXPr {
animation-delay: 0.1s;
}
.spinner_ZTLf {
animation-delay: 0.2s;
}
@keyframes spinner_8HQG {
0%,
57.14% {
animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
transform: translate(0);
}
28.57% {
animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
transform: translateY(-6px);
}
100% {
transform: translate(0);
}
}
</style><circle class="spinner_qM83" cx="4" cy="12" r="3" /><circle
class="spinner_qM83 spinner_oXPr"
cx="12"
cy="12"
r="3"
/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
>
{:else}
<div
class=" {rmsLevel * 100 > 4
? ' size-[4.5rem]'
: rmsLevel * 100 > 2
? ' size-16'
: rmsLevel * 100 > 1
? 'size-14'
: 'size-12'} transition-all bg-black dark:bg-white rounded-full"
/>
{/if}
<!-- navbar -->
</div>
{/if}
<div class="flex justify-center items-center flex-1 h-full w-full max-h-full">
{#if !camera}
{#if emoji}
<div
class=" transition-all rounded-full"
style="font-size:{rmsLevel * 100 > 4
? '13'
: rmsLevel * 100 > 2
? '12'
: rmsLevel * 100 > 1
? '11.5'
: '11'}rem;width:100%;text-align:center;"
>
{emoji}
</div>
{:else if loading}
<svg
class="size-44 text-gray-900 dark:text-gray-400"
viewBox="0 0 24 24"
fill="currentColor"
xmlns="http://www.w3.org/2000/svg"
><style>
.spinner_qM83 {
animation: spinner_8HQG 1.05s infinite;
}
.spinner_oXPr {
animation-delay: 0.1s;
}
.spinner_ZTLf {
animation-delay: 0.2s;
}
@keyframes spinner_8HQG {
0%,
57.14% {
animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
transform: translate(0);
}
28.57% {
animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
transform: translateY(-6px);
}
100% {
transform: translate(0);
}
}
</style><circle class="spinner_qM83" cx="4" cy="12" r="3" /><circle
class="spinner_qM83 spinner_oXPr"
cx="12"
cy="12"
r="3"
/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
>
{:else}
<div
class=" {rmsLevel * 100 > 4
? ' size-52'
: rmsLevel * 100 > 2
? 'size-48'
: rmsLevel * 100 > 1
? 'size-[11.5rem]'
: 'size-44'} transition-all bg-black dark:bg-white rounded-full"
/>
{/if}
{:else}
<div
class="relative flex video-container w-full max-h-full pt-2 pb-4 md:py-6 px-2 h-full"
>
<video
id="camera-feed"
autoplay
class="rounded-2xl h-full min-w-full object-cover object-center"
playsinline
/>
<canvas id="camera-canvas" style="display:none;" />
<div class=" absolute top-4 md:top-8 left-4">
<button
type="button"
class="p-1.5 text-white cursor-pointer backdrop-blur-xl bg-black/10 rounded-full"
on:click={() => {
stopCamera();
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="size-6"
>
<path
d="M5.28 4.22a.75.75 0 0 0-1.06 1.06L6.94 8l-2.72 2.72a.75.75 0 1 0 1.06 1.06L8 9.06l2.72 2.72a.75.75 0 1 0 1.06-1.06L9.06 8l2.72-2.72a.75.75 0 0 0-1.06-1.06L8 6.94 5.28 4.22Z"
/>
</svg>
</button>
</div>
</div>
{/if}
</div>
<div class="flex justify-between items-center pb-2 w-full">
<div>
{#if camera}
<VideoInputMenu
devices={videoInputDevices}
on:change={async (e) => {
console.log(e.detail);
selectedVideoInputDeviceId = e.detail;
await stopVideoStream();
await startVideoStream();
}}
>
<button class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900" type="button">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="size-5"
>
<path
fill-rule="evenodd"
d="M15.312 11.424a5.5 5.5 0 0 1-9.201 2.466l-.312-.311h2.433a.75.75 0 0 0 0-1.5H3.989a.75.75 0 0 0-.75.75v4.242a.75.75 0 0 0 1.5 0v-2.43l.31.31a7 7 0 0 0 11.712-3.138.75.75 0 0 0-1.449-.39Zm1.23-3.723a.75.75 0 0 0 .219-.53V2.929a.75.75 0 0 0-1.5 0V5.36l-.31-.31A7 7 0 0 0 3.239 8.188a.75.75 0 1 0 1.448.389A5.5 5.5 0 0 1 13.89 6.11l.311.31h-2.432a.75.75 0 0 0 0 1.5h4.243a.75.75 0 0 0 .53-.219Z"
clip-rule="evenodd"
/>
</svg>
</button>
</VideoInputMenu>
{:else}
<Tooltip content={$i18n.t('Camera')}>
<button
class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
type="button"
on:click={async () => {
await navigator.mediaDevices.getUserMedia({ video: true });
startCamera();
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="size-5"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M6.827 6.175A2.31 2.31 0 0 1 5.186 7.23c-.38.054-.757.112-1.134.175C2.999 7.58 2.25 8.507 2.25 9.574V18a2.25 2.25 0 0 0 2.25 2.25h15A2.25 2.25 0 0 0 21.75 18V9.574c0-1.067-.75-1.994-1.802-2.169a47.865 47.865 0 0 0-1.134-.175 2.31 2.31 0 0 1-1.64-1.055l-.822-1.316a2.192 2.192 0 0 0-1.736-1.039 48.774 48.774 0 0 0-5.232 0 2.192 2.192 0 0 0-1.736 1.039l-.821 1.316Z"
/>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M16.5 12.75a4.5 4.5 0 1 1-9 0 4.5 4.5 0 0 1 9 0ZM18.75 10.5h.008v.008h-.008V10.5Z"
/>
</svg>
</button>
</Tooltip>
{/if}
</div>
<div>
<button type="button">
<div class=" line-clamp-1 text-sm font-medium">
{#if loading}
{$i18n.t('Thinking...')}
{:else}
{$i18n.t('Listening...')}
{/if}
</div>
</button>
</div>
<div>
<button
class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
on:click={async () => {
showCallOverlay.set(false);
}}
type="button"
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="size-5"
>
<path
d="M6.28 5.22a.75.75 0 0 0-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 1 0 1.06 1.06L10 11.06l3.72 3.72a.75.75 0 1 0 1.06-1.06L11.06 10l3.72-3.72a.75.75 0 0 0-1.06-1.06L10 8.94 6.28 5.22Z"
/>
</svg>
</button>
</div>
</div>
</div>
</div>
</div>
{/if}
src/lib/components/chat/MessageInput/CallOverlay/VideoInputMenu.svelte 0 → 100644
View file @ 4ff17acc
<script lang="ts">
import { DropdownMenu } from 'bits-ui';
import { flyAndScale } from '$lib/utils/transitions';
import { getContext, createEventDispatcher } from 'svelte';
const i18n = getContext('i18n');
const dispatch = createEventDispatcher();
import Dropdown from '$lib/components/common/Dropdown.svelte';
export let onClose: Function = () => {};
export let devices: any;
let show = false;
</script>
<Dropdown
bind:show
on:change={(e) => {
if (e.detail === false) {
onClose();
}
}}
>
<slot />
<div slot="content">
<DropdownMenu.Content
class="w-full max-w-[180px] rounded-lg px-1 py-1.5 border border-gray-300/30 dark:border-gray-700/50 z-[9999] bg-white dark:bg-gray-900 dark:text-white shadow-sm"
sideOffset={6}
side="top"
align="start"
transition={flyAndScale}
>
{#each devices as device}
<DropdownMenu.Item
class="flex gap-2 items-center px-3 py-2 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
on:click={() => {
dispatch('change', device.deviceId);
}}
>
<div class="flex items-center">
<div class=" line-clamp-1">
{device?.label ?? 'Camera'}
</div>
</div>
</DropdownMenu.Item>
{/each}
</DropdownMenu.Content>
</div>
</Dropdown>
src/lib/components/chat/MessageInput/Documents.svelte
View file @ 4ff17acc
......@@ -101,18 +101,20 @@
</script>
{#if filteredItems.length > 0 || prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
<div class="md:px-2 mb-3 text-left w-full absolute bottom-0 left-0 right-0">
<div class="pl-1 pr-12 mb-3 text-left w-full absolute bottom-0 left-0 right-0">
<div class="flex w-full px-2">
<div class=" bg-gray-100 dark:bg-gray-700 w-10 rounded-l-xl text-center">
<div class=" text-lg font-semibold mt-2">#</div>
</div>
<div class="max-h-60 flex flex-col w-full rounded-r-xl bg-white">
<div
class="max-h-60 flex flex-col w-full rounded-r-xl bg-white dark:bg-gray-850 dark:text-gray-100"
>
<div class="m-1 overflow-y-auto p-1 rounded-r-xl space-y-0.5">
{#each filteredItems as doc, docIdx}
<button
class=" px-3 py-1.5 rounded-xl w-full text-left {docIdx === selectedIdx
? ' bg-gray-100 selected-command-option-button'
? ' bg-gray-100 dark:bg-gray-600 dark:text-gray-100 selected-command-option-button'
: ''}"
type="button"
on:click={() => {
......@@ -126,17 +128,19 @@
on:focus={() => {}}
>
{#if doc.type === 'collection'}
<div class=" font-medium text-black line-clamp-1">
<div class=" font-medium text-black dark:text-gray-100 line-clamp-1">
{doc?.title ?? `#${doc.name}`}
</div>
<div class=" text-xs text-gray-600 line-clamp-1">{$i18n.t('Collection')}</div>
<div class=" text-xs text-gray-600 dark:text-gray-100 line-clamp-1">
{$i18n.t('Collection')}
</div>
{:else}
<div class=" font-medium text-black line-clamp-1">
<div class=" font-medium text-black dark:text-gray-100 line-clamp-1">
#{doc.name} ({doc.filename})
</div>
<div class=" text-xs text-gray-600 line-clamp-1">
<div class=" text-xs text-gray-600 dark:text-gray-100 line-clamp-1">
{doc.title}
</div>
{/if}
......
src/lib/components/chat/MessageInput/InputMenu.svelte
View file @ 4ff17acc
......@@ -4,24 +4,33 @@
import { getContext } from 'svelte';
import Dropdown from '$lib/components/common/Dropdown.svelte';
import GarbageBin from '$lib/components/icons/GarbageBin.svelte';
import Pencil from '$lib/components/icons/Pencil.svelte';
import Tooltip from '$lib/components/common/Tooltip.svelte';
import Tags from '$lib/components/chat/Tags.svelte';
import Share from '$lib/components/icons/Share.svelte';
import ArchiveBox from '$lib/components/icons/ArchiveBox.svelte';
import DocumentArrowUpSolid from '$lib/components/icons/DocumentArrowUpSolid.svelte';
import Switch from '$lib/components/common/Switch.svelte';
import GlobeAltSolid from '$lib/components/icons/GlobeAltSolid.svelte';
import { config } from '$lib/stores';
import WrenchSolid from '$lib/components/icons/WrenchSolid.svelte';
const i18n = getContext('i18n');
export let uploadFilesHandler: Function;
export let selectedToolIds: string[] = [];
export let webSearchEnabled: boolean;
export let tools = {};
export let onClose: Function;
$: tools = Object.fromEntries(
Object.keys(tools).map((toolId) => [
toolId,
{
...tools[toolId],
enabled: selectedToolIds.includes(toolId)
}
])
);
let show = false;
</script>
......@@ -39,20 +48,48 @@
<div slot="content">
<DropdownMenu.Content
class="w-full max-w-[190px] rounded-xl px-1 py-1 border-gray-300/30 dark:border-gray-700/50 z-50 bg-white dark:bg-gray-850 dark:text-white shadow"
class="w-full max-w-[200px] rounded-xl px-1 py-1 border-gray-300/30 dark:border-gray-700/50 z-50 bg-white dark:bg-gray-850 dark:text-white shadow"
sideOffset={15}
alignOffset={-8}
side="top"
align="start"
transition={flyAndScale}
>
{#if Object.keys(tools).length > 0}
<div class=" max-h-28 overflow-y-auto scrollbar-hidden">
{#each Object.keys(tools) as toolId}
<div
class="flex gap-2 items-center px-3 py-2 text-sm font-medium cursor-pointer rounded-xl"
>
<div class="flex-1 flex items-center gap-2">
<WrenchSolid />
<Tooltip content={tools[toolId]?.description ?? ''} className="flex-1">
<div class=" line-clamp-1">{tools[toolId].name}</div>
</Tooltip>
</div>
<Switch
bind:state={tools[toolId].enabled}
on:change={(e) => {
selectedToolIds = e.detail
? [...selectedToolIds, toolId]
: selectedToolIds.filter((id) => id !== toolId);
}}
/>
</div>
{/each}
</div>
<hr class="border-gray-100 dark:border-gray-800 my-1" />
{/if}
{#if $config?.features?.enable_web_search}
<div
class="flex gap-2 items-center px-3 py-2 text-sm font-medium cursor-pointer rounded-xl"
>
<div class="flex-1 flex items-center gap-2">
<GlobeAltSolid />
<div class="flex items-center">{$i18n.t('Web Search')}</div>
<div class=" line-clamp-1">{$i18n.t('Web Search')}</div>
</div>
<Switch bind:state={webSearchEnabled} />
......@@ -68,7 +105,7 @@
}}
>
<DocumentArrowUpSolid />
<div class="flex items-center">{$i18n.t('Upload Files')}</div>
<div class=" line-clamp-1">{$i18n.t('Upload Files')}</div>
</DropdownMenu.Item>
</DropdownMenu.Content>
</div>
......
src/lib/components/chat/MessageInput/Models.svelte
View file @ 4ff17acc
......@@ -133,18 +133,18 @@
{#if prompt.charAt(0) === '@'}
{#if filteredModels.length > 0}
<div class="md:px-2 mb-3 text-left w-full absolute bottom-0 left-0 right-0">
<div class="pl-1 pr-12 mb-3 text-left w-full absolute bottom-0 left-0 right-0">
<div class="flex w-full px-2">
<div class=" bg-gray-100 dark:bg-gray-700 w-10 rounded-l-xl text-center">
<div class=" text-lg font-semibold mt-2">@</div>
</div>
<div class="max-h-60 flex flex-col w-full rounded-r-xl bg-white">
<div class="max-h-60 flex flex-col w-full rounded-r-xl bg-white dark:bg-gray-850">
<div class="m-1 overflow-y-auto p-1 rounded-r-xl space-y-0.5">
{#each filteredModels as model, modelIdx}
<button
class=" px-3 py-1.5 rounded-xl w-full text-left {modelIdx === selectedIdx
? ' bg-gray-100 selected-command-option-button'
? ' bg-gray-100 dark:bg-gray-600 selected-command-option-button'
: ''}"
type="button"
on:click={() => {
......@@ -155,7 +155,7 @@
}}
on:focus={() => {}}
>
<div class=" font-medium text-black line-clamp-1">
<div class=" font-medium text-black dark:text-gray-100 line-clamp-1">
{model.name}
</div>
......
src/lib/components/chat/MessageInput/PromptCommands.svelte
View file @ 4ff17acc
......@@ -6,6 +6,7 @@
const i18n = getContext('i18n');
export let files;
export let prompt = '';
let selectedCommandIdx = 0;
let filteredPromptCommands = [];
......@@ -35,6 +36,32 @@
return '{{CLIPBOARD}}';
});
console.log(clipboardText);
const clipboardItems = await navigator.clipboard.read();
let imageUrl = null;
for (const item of clipboardItems) {
// Check for known image types
for (const type of item.types) {
if (type.startsWith('image/')) {
const blob = await item.getType(type);
imageUrl = URL.createObjectURL(blob);
console.log(`Image URL (${type}): ${imageUrl}`);
}
}
}
if (imageUrl) {
files = [
...files,
{
type: 'image',
url: imageUrl
}
];
}
text = command.content.replaceAll('{{CLIPBOARD}}', clipboardText);
}
......@@ -61,18 +88,18 @@
</script>
{#if filteredPromptCommands.length > 0}
<div class="md:px-2 mb-3 text-left w-full absolute bottom-0 left-0 right-0">
<div class="pl-1 pr-12 mb-3 text-left w-full absolute bottom-0 left-0 right-0">
<div class="flex w-full px-2">
<div class=" bg-gray-100 dark:bg-gray-700 w-10 rounded-l-xl text-center">
<div class=" text-lg font-semibold mt-2">/</div>
</div>
<div class="max-h-60 flex flex-col w-full rounded-r-xl bg-white">
<div class="max-h-60 flex flex-col w-full rounded-r-xl bg-white dark:bg-gray-850">
<div class="m-1 overflow-y-auto p-1 rounded-r-xl space-y-0.5">
{#each filteredPromptCommands as command, commandIdx}
<button
class=" px-3 py-1.5 rounded-xl w-full text-left {commandIdx === selectedCommandIdx
? ' bg-gray-100 selected-command-option-button'
? ' bg-gray-100 dark:bg-gray-600 selected-command-option-button'
: ''}"
type="button"
on:click={() => {
......@@ -83,11 +110,11 @@
}}
on:focus={() => {}}
>
<div class=" font-medium text-black">
<div class=" font-medium text-black dark:text-gray-100">
{command.command}
</div>
<div class=" text-xs text-gray-600">
<div class=" text-xs text-gray-600 dark:text-gray-100">
{command.title}
</div>
</button>
......@@ -95,7 +122,7 @@
</div>
<div
class=" px-2 pb-1 text-xs text-gray-600 bg-white rounded-br-xl flex items-center space-x-1"
class=" px-2 pb-1 text-xs text-gray-600 dark:text-gray-100 bg-white dark:bg-gray-850 rounded-br-xl flex items-center space-x-1"
>
<div>
<svg
......
src/lib/components/chat/MessageInput/VoiceRecording.svelte 0 → 100644
View file @ 4ff17acc
<script lang="ts">
import { toast } from 'svelte-sonner';
import { createEventDispatcher, tick, getContext } from 'svelte';
import { config, settings } from '$lib/stores';
import { blobToFile, calculateSHA256, findWordIndices } from '$lib/utils';
import { transcribeAudio } from '$lib/apis/audio';
const i18n = getContext('i18n');
const dispatch = createEventDispatcher();
export let recording = false;
let loading = false;
let confirmed = false;
let durationSeconds = 0;
let durationCounter = null;
let transcription = '';
const startDurationCounter = () => {
durationCounter = setInterval(() => {
durationSeconds++;
}, 1000);
};
const stopDurationCounter = () => {
clearInterval(durationCounter);
durationSeconds = 0;
};
$: if (recording) {
startRecording();
} else {
stopRecording();
}
const formatSeconds = (seconds) => {
const minutes = Math.floor(seconds / 60);
const remainingSeconds = seconds % 60;
const formattedSeconds = remainingSeconds < 10 ? `0${remainingSeconds}` : remainingSeconds;
return `${minutes}:${formattedSeconds}`;
};
let speechRecognition;
let mediaRecorder;
let audioChunks = [];
const MIN_DECIBELS = -45;
const VISUALIZER_BUFFER_LENGTH = 300;
let visualizerData = Array(VISUALIZER_BUFFER_LENGTH).fill(0);
// Function to calculate the RMS level from time domain data
const calculateRMS = (data: Uint8Array) => {
let sumSquares = 0;
for (let i = 0; i < data.length; i++) {
const normalizedValue = (data[i] - 128) / 128; // Normalize the data
sumSquares += normalizedValue * normalizedValue;
}
return Math.sqrt(sumSquares / data.length);
};
const normalizeRMS = (rms) => {
rms = rms * 10;
const exp = 1.5; // Adjust exponent value; values greater than 1 expand larger numbers more and compress smaller numbers more
const scaledRMS = Math.pow(rms, exp);
// Scale between 0.01 (1%) and 1.0 (100%)
return Math.min(1.0, Math.max(0.01, scaledRMS));
};
const analyseAudio = (stream) => {
const audioContext = new AudioContext();
const audioStreamSource = audioContext.createMediaStreamSource(stream);
const analyser = audioContext.createAnalyser();
analyser.minDecibels = MIN_DECIBELS;
audioStreamSource.connect(analyser);
const bufferLength = analyser.frequencyBinCount;
const domainData = new Uint8Array(bufferLength);
const timeDomainData = new Uint8Array(analyser.fftSize);
let lastSoundTime = Date.now();
const detectSound = () => {
const processFrame = () => {
if (!recording || loading) return;
if (recording && !loading) {
analyser.getByteTimeDomainData(timeDomainData);
analyser.getByteFrequencyData(domainData);
// Calculate RMS level from time domain data
const rmsLevel = calculateRMS(timeDomainData);
// Push the calculated decibel level to visualizerData
visualizerData.push(normalizeRMS(rmsLevel));
// Ensure visualizerData array stays within the buffer length
if (visualizerData.length >= VISUALIZER_BUFFER_LENGTH) {
visualizerData.shift();
}
visualizerData = visualizerData;
// if (domainData.some((value) => value > 0)) {
// lastSoundTime = Date.now();
// }
// if (recording && Date.now() - lastSoundTime > 3000) {
// if ($settings?.speechAutoSend ?? false) {
// confirmRecording();
// }
// }
}
window.requestAnimationFrame(processFrame);
};
window.requestAnimationFrame(processFrame);
};
detectSound();
};
const transcribeHandler = async (audioBlob) => {
// Create a blob from the audio chunks
await tick();
const file = blobToFile(audioBlob, 'recording.wav');
const res = await transcribeAudio(localStorage.token, file).catch((error) => {
toast.error(error);
return null;
});
if (res) {
console.log(res.text);
dispatch('confirm', res.text);
}
};
const saveRecording = (blob) => {
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
document.body.appendChild(a);
a.style = 'display: none';
a.href = url;
a.download = 'recording.wav';
a.click();
window.URL.revokeObjectURL(url);
};
const startRecording = async () => {
startDurationCounter();
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
mediaRecorder = new MediaRecorder(stream);
mediaRecorder.onstart = () => {
console.log('Recording started');
audioChunks = [];
analyseAudio(stream);
};
mediaRecorder.ondataavailable = (event) => audioChunks.push(event.data);
mediaRecorder.onstop = async () => {
console.log('Recording stopped');
if (($settings?.audio?.stt?.engine ?? '') === 'web') {
audioChunks = [];
} else {
if (confirmed) {
const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
await transcribeHandler(audioBlob);
confirmed = false;
loading = false;
}
audioChunks = [];
recording = false;
}
};
mediaRecorder.start();
if ($config.audio.stt.engine === 'web' || ($settings?.audio?.stt?.engine ?? '') === 'web') {
if ('SpeechRecognition' in window || 'webkitSpeechRecognition' in window) {
// Create a SpeechRecognition object
speechRecognition = new (window.SpeechRecognition || window.webkitSpeechRecognition)();
// Set continuous to true for continuous recognition
speechRecognition.continuous = true;
// Set the timeout for turning off the recognition after inactivity (in milliseconds)
const inactivityTimeout = 2000; // 3 seconds
let timeoutId;
// Start recognition
speechRecognition.start();
// Event triggered when speech is recognized
speechRecognition.onresult = async (event) => {
// Clear the inactivity timeout
clearTimeout(timeoutId);
// Handle recognized speech
console.log(event);
const transcript = event.results[Object.keys(event.results).length - 1][0].transcript;
transcription = `${transcription}${transcript}`;
await tick();
document.getElementById('chat-textarea')?.focus();
// Restart the inactivity timeout
timeoutId = setTimeout(() => {
console.log('Speech recognition turned off due to inactivity.');
speechRecognition.stop();
}, inactivityTimeout);
};
// Event triggered when recognition is ended
speechRecognition.onend = function () {
// Restart recognition after it ends
console.log('recognition ended');
confirmRecording();
dispatch('confirm', transcription);
confirmed = false;
loading = false;
};
// Event triggered when an error occurs
speechRecognition.onerror = function (event) {
console.log(event);
toast.error($i18n.t(`Speech recognition error: {{error}}`, { error: event.error }));
dispatch('cancel');
stopRecording();
};
}
}
};
const stopRecording = async () => {
if (recording && mediaRecorder) {
await mediaRecorder.stop();
}
stopDurationCounter();
audioChunks = [];
};
const confirmRecording = async () => {
loading = true;
confirmed = true;
if (recording && mediaRecorder) {
await mediaRecorder.stop();
}
clearInterval(durationCounter);
};
</script>
<div
class="{loading
? ' bg-gray-100/50 dark:bg-gray-850/50'
: 'bg-indigo-300/10 dark:bg-indigo-500/10 '} rounded-full flex p-2.5"
>
<div class="flex items-center mr-1">
<button
type="button"
class="p-1.5
{loading
? ' bg-gray-200 dark:bg-gray-700/50'
: 'bg-indigo-400/20 text-indigo-600 dark:text-indigo-300 '}
rounded-full"
on:click={async () => {
dispatch('cancel');
stopRecording();
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="3"
stroke="currentColor"
class="size-4"
>
<path stroke-linecap="round" stroke-linejoin="round" d="M6 18 18 6M6 6l12 12" />
</svg>
</button>
</div>
<div
class="flex flex-1 self-center items-center justify-between ml-2 mx-1 overflow-hidden h-6"
dir="rtl"
>
<div class="flex-1 flex items-center gap-0.5 h-6">
{#each visualizerData.slice().reverse() as rms}
<div
class="w-[2px]
{loading
? ' bg-gray-500 dark:bg-gray-400 '
: 'bg-indigo-500 dark:bg-indigo-400 '}
inline-block h-full"
style="height: {Math.min(100, Math.max(14, rms * 100))}%;"
/>
{/each}
</div>
</div>
<div class=" mx-1.5 pr-1 flex justify-center items-center">
<div
class="text-sm
{loading ? ' text-gray-500 dark:text-gray-400 ' : ' text-indigo-400 '}
font-medium flex-1 mx-auto text-center"
>
{formatSeconds(durationSeconds)}
</div>
</div>
<div class="flex items-center mr-1">
{#if loading}
<div class=" text-gray-500 rounded-full cursor-not-allowed">
<svg
width="24"
height="24"
viewBox="0 0 24 24"
xmlns="http://www.w3.org/2000/svg"
fill="currentColor"
><style>
.spinner_OSmW {
transform-origin: center;
animation: spinner_T6mA 0.75s step-end infinite;
}
@keyframes spinner_T6mA {
8.3% {
transform: rotate(30deg);
}
16.6% {
transform: rotate(60deg);
}
25% {
transform: rotate(90deg);
}
33.3% {
transform: rotate(120deg);
}
41.6% {
transform: rotate(150deg);
}
50% {
transform: rotate(180deg);
}
58.3% {
transform: rotate(210deg);
}
66.6% {
transform: rotate(240deg);
}
75% {
transform: rotate(270deg);
}
83.3% {
transform: rotate(300deg);
}
91.6% {
transform: rotate(330deg);
}
100% {
transform: rotate(360deg);
}
}
</style><g class="spinner_OSmW"
><rect x="11" y="1" width="2" height="5" opacity=".14" /><rect
x="11"
y="1"
width="2"
height="5"
transform="rotate(30 12 12)"
opacity=".29"
/><rect
x="11"
y="1"
width="2"
height="5"
transform="rotate(60 12 12)"
opacity=".43"
/><rect
x="11"
y="1"
width="2"
height="5"
transform="rotate(90 12 12)"
opacity=".57"
/><rect
x="11"
y="1"
width="2"
height="5"
transform="rotate(120 12 12)"
opacity=".71"
/><rect
x="11"
y="1"
width="2"
height="5"
transform="rotate(150 12 12)"
opacity=".86"
/><rect x="11" y="1" width="2" height="5" transform="rotate(180 12 12)" /></g
></svg
>
</div>
{:else}
<button
type="button"
class="p-1.5 bg-indigo-500 text-white dark:bg-indigo-500 dark:text-blue-950 rounded-full"
on:click={async () => {
await confirmRecording();
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="2.5"
stroke="currentColor"
class="size-4"
>
<path stroke-linecap="round" stroke-linejoin="round" d="m4.5 12.75 6 6 9-13.5" />
</svg>
</button>
{/if}
</div>
</div>
<style>
.visualizer {
display: flex;
height: 100%;
}
.visualizer-bar {
width: 2px;
background-color: #4a5aba; /* or whatever color you need */
}
</style>
src/lib/components/chat/Messages/CompareMessages.svelte
View file @ 4ff17acc
......@@ -109,7 +109,7 @@
class=" snap-center min-w-80 w-full max-w-full m-1 border {history.messages[
currentMessageId
].model === model
? 'border-gray-100 dark:border-gray-700 border-[1.5px]'
? 'border-gray-100 dark:border-gray-850 border-[1.5px]'
: 'border-gray-50 dark:border-gray-850 '} transition p-5 rounded-3xl"
on:click={() => {
currentMessageId = groupedMessages[model].messages[groupedMessagesIdx[model]].id;
......
src/lib/components/chat/Messages/Placeholder.svelte
View file @ 4ff17acc
<script lang="ts">
import { WEBUI_BASE_URL } from '$lib/constants';
import { marked } from 'marked';
import { config, user, models as _models } from '$lib/stores';
import { onMount, getContext } from 'svelte';
import { blur, fade } from 'svelte/transition';
import Suggestions from '../MessageInput/Suggestions.svelte';
import { sanitizeResponseContent } from '$lib/utils';
const i18n = getContext('i18n');
......@@ -29,7 +32,7 @@
</script>
{#key mounted}
<div class="m-auto w-full max-w-6xl px-8 lg:px-24 pb-16">
<div class="m-auto w-full max-w-6xl px-8 lg:px-24 pb-10">
<div class="flex justify-start">
<div class="flex -space-x-4 mb-1" in:fade={{ duration: 200 }}>
{#each models as model, modelIdx}
......@@ -65,8 +68,12 @@
<div in:fade={{ duration: 200, delay: 200 }}>
{#if models[selectedModelIdx]?.info?.meta?.description ?? null}
<div class="mt-0.5 text-base font-normal text-gray-500 dark:text-gray-400 line-clamp-3">
{models[selectedModelIdx]?.info?.meta?.description}
<div
class="mt-0.5 text-base font-normal text-gray-500 dark:text-gray-400 line-clamp-3 markdown"
>
{@html marked.parse(
sanitizeResponseContent(models[selectedModelIdx]?.info?.meta?.description)
)}
</div>
{#if models[selectedModelIdx]?.info?.meta?.user}
<div class="mt-0.5 text-sm font-normal text-gray-400 dark:text-gray-500">
......@@ -85,7 +92,7 @@
</div>
{/if}
{:else}
<div class=" font-medium text-gray-400 dark:text-gray-500">
<div class=" font-medium text-gray-400 dark:text-gray-500 line-clamp-1">
{$i18n.t('How can I help you today?')}
</div>
{/if}
......
src/lib/components/chat/Messages/RateComment.svelte
View file @ 4ff17acc
......@@ -46,8 +46,8 @@
}
onMount(() => {
selectedReason = message.annotation.reason;
comment = message.annotation.comment;
selectedReason = message?.annotation?.reason ?? '';
comment = message?.annotation?.comment ?? '';
loadReasons();
});
......
src/lib/components/chat/Messages/ResponseMessage.svelte
View file @ 4ff17acc
......@@ -211,82 +211,98 @@
speaking = null;
speakingIdx = null;
} else {
speaking = true;
if ($settings?.audio?.TTSEngine === 'openai') {
loadingSpeech = true;
const sentences = extractSentences(message.content).reduce((mergedTexts, currentText) => {
const lastIndex = mergedTexts.length - 1;
if (lastIndex >= 0) {
const previousText = mergedTexts[lastIndex];
const wordCount = previousText.split(/\s+/).length;
if (wordCount < 2) {
mergedTexts[lastIndex] = previousText + ' ' + currentText;
if ((message?.content ?? '').trim() !== '') {
speaking = true;
if ($config.audio.tts.engine === 'openai') {
loadingSpeech = true;
const sentences = extractSentences(message.content).reduce((mergedTexts, currentText) => {
const lastIndex = mergedTexts.length - 1;
if (lastIndex >= 0) {
const previousText = mergedTexts[lastIndex];
const wordCount = previousText.split(/\s+/).length;
if (wordCount < 2) {
mergedTexts[lastIndex] = previousText + ' ' + currentText;
} else {
mergedTexts.push(currentText);
}
} else {
mergedTexts.push(currentText);
}
} else {
mergedTexts.push(currentText);
}
return mergedTexts;
}, []);
console.log(sentences);
sentencesAudio = sentences.reduce((a, e, i, arr) => {
a[i] = null;
return a;
}, {});
let lastPlayedAudioPromise = Promise.resolve(); // Initialize a promise that resolves immediately
for (const [idx, sentence] of sentences.entries()) {
const res = await synthesizeOpenAISpeech(
localStorage.token,
$settings?.audio?.speaker,
sentence,
$settings?.audio?.model
).catch((error) => {
toast.error(error);
speaking = null;
loadingSpeech = false;
return null;
});
if (res) {
const blob = await res.blob();
const blobUrl = URL.createObjectURL(blob);
const audio = new Audio(blobUrl);
sentencesAudio[idx] = audio;
loadingSpeech = false;
lastPlayedAudioPromise = lastPlayedAudioPromise.then(() => playAudio(idx));
}
}
} else {
let voices = [];
const getVoicesLoop = setInterval(async () => {
voices = await speechSynthesis.getVoices();
if (voices.length > 0) {
clearInterval(getVoicesLoop);
return mergedTexts;
}, []);
console.log(sentences);
sentencesAudio = sentences.reduce((a, e, i, arr) => {
a[i] = null;
return a;
}, {});
const voice =
voices?.filter((v) => v.name === $settings?.audio?.speaker)?.at(0) ?? undefined;
let lastPlayedAudioPromise = Promise.resolve(); // Initialize a promise that resolves immediately
const speak = new SpeechSynthesisUtterance(message.content);
for (const [idx, sentence] of sentences.entries()) {
const res = await synthesizeOpenAISpeech(
localStorage.token,
$settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice,
sentence
).catch((error) => {
toast.error(error);
speak.onend = () => {
speaking = null;
if ($settings.conversationMode) {
document.getElementById('voice-input-button')?.click();
}
};
speak.voice = voice;
speechSynthesis.speak(speak);
loadingSpeech = false;
return null;
});
if (res) {
const blob = await res.blob();
const blobUrl = URL.createObjectURL(blob);
const audio = new Audio(blobUrl);
sentencesAudio[idx] = audio;
loadingSpeech = false;
lastPlayedAudioPromise = lastPlayedAudioPromise.then(() => playAudio(idx));
}
}
}, 100);
} else {
let voices = [];
const getVoicesLoop = setInterval(async () => {
voices = await speechSynthesis.getVoices();
if (voices.length > 0) {
clearInterval(getVoicesLoop);
const voice =
voices
?.filter(
(v) =>
v.voiceURI === ($settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice)
)
?.at(0) ?? undefined;
console.log(voice);
const speak = new SpeechSynthesisUtterance(message.content);
console.log(speak);
speak.onend = () => {
speaking = null;
if ($settings.conversationMode) {
document.getElementById('voice-input-button')?.click();
}
};
if (voice) {
speak.voice = voice;
}
speechSynthesis.speak(speak);
}
}, 100);
}
} else {
toast.error('No content to speak');
}
}
};
......@@ -404,26 +420,29 @@
class="prose chat-{message.role} w-full max-w-full dark:prose-invert prose-headings:my-0 prose-headings:-mb-4 prose-p:m-0 prose-p:-mb-6 prose-pre:my-0 prose-table:my-0 prose-blockquote:my-0 prose-img:my-0 prose-ul:-my-4 prose-ol:-my-4 prose-li:-my-3 prose-ul:-mb-6 prose-ol:-mb-8 prose-ol:p-0 prose-li:-mb-4 whitespace-pre-line"
>
<div>
{#if message?.status}
{#if (message?.statusHistory ?? [...(message?.status ? [message?.status] : [])]).length > 0}
{@const status = (
message?.statusHistory ?? [...(message?.status ? [message?.status] : [])]
).at(-1)}
<div class="flex items-center gap-2 pt-1 pb-1">
{#if message?.status?.done === false}
{#if status.done === false}
<div class="">
<Spinner className="size-4" />
</div>
{/if}
{#if message?.status?.action === 'web_search' && message?.status?.urls}
<WebSearchResults urls={message?.status?.urls}>
{#if status?.action === 'web_search' && status?.urls}
<WebSearchResults {status}>
<div class="flex flex-col justify-center -space-y-0.5">
<div class="text-base line-clamp-1 text-wrap">
{message.status.description}
{status?.description}
</div>
</div>
</WebSearchResults>
{:else}
<div class="flex flex-col justify-center -space-y-0.5">
<div class=" text-gray-500 dark:text-gray-500 text-base line-clamp-1 text-wrap">
{message.status.description}
{status?.description}
</div>
</div>
{/if}
......@@ -757,7 +776,7 @@
</Tooltip>
{#if $config?.features.enable_image_generation && !readOnly}
<Tooltip content="Generate Image" placement="bottom">
<Tooltip content={$i18n.t('Generate Image')} placement="bottom">
<button
class="{isLastMessage
? 'visible'
......@@ -857,8 +876,8 @@
<button
class="{isLastMessage
? 'visible'
: 'invisible group-hover:visible'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg {message
?.annotation?.rating === 1
: 'invisible group-hover:visible'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg {(message
?.annotation?.rating ?? null) === 1
? 'bg-gray-100 dark:bg-gray-800'
: ''} dark:hover:text-white hover:text-black transition"
on:click={() => {
......@@ -892,8 +911,8 @@
<button
class="{isLastMessage
? 'visible'
: 'invisible group-hover:visible'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg {message
?.annotation?.rating === -1
: 'invisible group-hover:visible'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg {(message
?.annotation?.rating ?? null) === -1
? 'bg-gray-100 dark:bg-gray-800'
: ''} dark:hover:text-white hover:text-black transition"
on:click={() => {
......@@ -963,6 +982,7 @@
? 'visible'
: 'invisible group-hover:visible'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg dark:hover:text-white hover:text-black transition regenerate-response-button"
on:click={() => {
showRateComment = false;
regenerateResponse(message);
}}
>
......
src/lib/components/chat/Messages/ResponseMessage/WebSearchResults.svelte
View file @ 4ff17acc
<script lang="ts">
import ChevronDown from '$lib/components/icons/ChevronDown.svelte';
import ChevronUp from '$lib/components/icons/ChevronUp.svelte';
import MagnifyingGlass from '$lib/components/icons/MagnifyingGlass.svelte';
import { Collapsible } from 'bits-ui';
import { slide } from 'svelte/transition';
export let urls = [];
export let status = { urls: [], query: '' };
let state = false;
</script>
......@@ -27,11 +28,45 @@
class=" text-sm border border-gray-300/30 dark:border-gray-700/50 rounded-xl"
transition={slide}
>
{#each urls as url, urlIdx}
{#if status?.query}
<a
href="https://www.google.com/search?q={status.query}"
target="_blank"
class="flex w-full items-center p-3 px-4 border-b border-gray-300/30 dark:border-gray-700/50 group/item justify-between font-normal text-gray-800 dark:text-gray-300 no-underline"
>
<div class="flex gap-2 items-center">
<MagnifyingGlass />
<div class=" line-clamp-1">
{status.query}
</div>
</div>
<div
class=" ml-1 text-white dark:text-gray-900 group-hover/item:text-gray-600 dark:group-hover/item:text-white transition"
>
<!-- -->
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="size-4"
>
<path
fill-rule="evenodd"
d="M4.22 11.78a.75.75 0 0 1 0-1.06L9.44 5.5H5.75a.75.75 0 0 1 0-1.5h5.5a.75.75 0 0 1 .75.75v5.5a.75.75 0 0 1-1.5 0V6.56l-5.22 5.22a.75.75 0 0 1-1.06 0Z"
clip-rule="evenodd"
/>
</svg>
</div>
</a>
{/if}
{#each status.urls as url, urlIdx}
<a
href={url}
target="_blank"
class="flex w-full items-center p-3 px-4 {urlIdx === urls.length - 1
class="flex w-full items-center p-3 px-4 {urlIdx === status.urls.length - 1
? ''
: 'border-b border-gray-300/30 dark:border-gray-700/50'} group/item justify-between font-normal text-gray-800 dark:text-gray-300"
>
......
src/lib/components/chat/ModelSelector/Selector.svelte
View file @ 4ff17acc
<script lang="ts">
import { DropdownMenu } from 'bits-ui';
import { marked } from 'marked';
import { flyAndScale } from '$lib/utils/transitions';
import { createEventDispatcher, onMount, getContext, tick } from 'svelte';
......@@ -152,6 +153,7 @@
toast.error(error);
// opts.callback({ success: false, error, modelName: opts.modelName });
break;
}
}
......@@ -219,7 +221,7 @@
<DropdownMenu.Content
class=" z-40 {$mobile
? `w-full`
: `${className}`} max-w-[calc(100vw-1rem)] justify-start rounded-xl bg-white dark:bg-gray-850 dark:text-white shadow-lg border border-gray-300/30 dark:border-gray-700/50 outline-none "
: `${className}`} max-w-[calc(100vw-1rem)] justify-start rounded-xl bg-white dark:bg-gray-850 dark:text-white shadow-lg border border-gray-300/30 dark:border-gray-850/50 outline-none "
transition={flyAndScale}
side={$mobile ? 'bottom' : 'bottom-start'}
sideOffset={4}
......@@ -265,7 +267,7 @@
</div>
{/if}
<div class="flex items-center gap-2">
<div class="flex items-center">
<div class="flex items-center min-w-fit">
<div class="line-clamp-1">
{item.label}
</div>
......@@ -332,9 +334,12 @@
{#if item.model?.info?.meta?.description}
<Tooltip
content={`${sanitizeResponseContent(
item.model?.info?.meta?.description
).replaceAll('\n', '<br>')}`}
content={`${marked.parse(
sanitizeResponseContent(item.model?.info?.meta?.description).replaceAll(
'\n',
'<br>'
)
)}`}
>
<div class="">
<svg
......
src/lib/components/chat/Settings/About.svelte
View file @ 4ff17acc
......@@ -92,7 +92,7 @@
</div>
{#if ollamaVersion}
<hr class=" dark:border-gray-700" />
<hr class=" dark:border-gray-850" />
<div>
<div class=" mb-2.5 text-sm font-medium">{$i18n.t('Ollama Version')}</div>
......@@ -104,7 +104,7 @@
</div>
{/if}
<hr class=" dark:border-gray-700" />
<hr class=" dark:border-gray-850" />
<div class="flex space-x-1">
<a href="https://discord.gg/5rJgQTnV4s" target="_blank">
......
src/lib/components/chat/Settings/Account.svelte
View file @ 4ff17acc
......@@ -234,7 +234,7 @@
<UpdatePassword />
</div>
<hr class=" dark:border-gray-700 my-4" />
<hr class=" dark:border-gray-850 my-4" />
<div class="flex justify-between items-center text-sm">
<div class=" font-medium">{$i18n.t('API keys')}</div>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment