Skip to content

GitLab

Commit 2e4f060e authored by Timothy J. Baek's avatar Timothy J. Baek
Browse files

feat: emoji call

parent 53858c9b
Show whitespace changes
Inline Side-by-side
Showing with 112 additions and 87 deletions
src/lib/apis/index.ts
View file @ 2e4f060e
......@@ -242,7 +242,15 @@ export const generateEmoji = async (
throw error;
}
return res?.choices[0]?.message?.content.replace(/["']/g, '') ?? null;
const response = res?.choices[0]?.message?.content.replace(/["']/g, '') ?? null;
if (response) {
if (/\p{Extended_Pictographic}/u.test(response)) {
return response.match(/\p{Extended_Pictographic}/gu)[0];
}
}
return null;
};
export const generateSearchQuery = async (
......
src/lib/components/chat/MessageInput/CallOverlay.svelte
View file @ 2e4f060e
......@@ -338,7 +338,7 @@
speechSynthesis.speak(currentUtterance);
currentUtterance.onend = async (e) => {
await new Promise((r) => setTimeout(r, 100));
await new Promise((r) => setTimeout(r, 200));
resolve(e);
};
}
......@@ -402,9 +402,20 @@
// Audio cache map where key is the content and value is the Audio object.
const audioCache = new Map();
const emojiCache = new Map();
const fetchAudio = async (content) => {
if (!audioCache.has(content)) {
try {
// Set the emoji for the content if needed
if ($settings?.showEmojiInCall ?? false) {
const emoji = await generateEmoji(localStorage.token, modelId, content, chatId);
if (emoji) {
emojiCache.set(content, emoji);
}
}
if ($config.audio.tts.engine !== '') {
const res = await synthesizeOpenAISpeech(
localStorage.token,
$settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice,
......@@ -419,10 +430,14 @@
const blobUrl = URL.createObjectURL(blob);
audioCache.set(content, new Audio(blobUrl));
}
} else {
audioCache.set(content, true);
}
} catch (error) {
console.error('Error synthesizing speech:', error);
}
}
return audioCache.get(content);
};
......@@ -436,6 +451,15 @@
if (audioCache.has(content)) {
// If content is available in the cache, play it
// Set the emoji for the content if available
if (($settings?.showEmojiInCall ?? false) && emojiCache.has(content)) {
emoji = emojiCache.get(content);
} else {
emoji = null;
}
if ($config.audio.tts.engine !== '') {
try {
console.log(
'%c%s',
......@@ -450,6 +474,9 @@
} catch (error) {
console.error('Error playing audio:', error);
}
} else {
await speakSpeechSynthesisHandler(content);
}
} else {
// If not available in the cache, push it back to the queue and delay
messages[id].unshift(content); // Re-queue the content at the start
......@@ -475,8 +502,6 @@
chatStreaming = true;
if ($config.audio.tts.engine !== '') {
// set currentMessageId to id
if (currentMessageId !== id) {
console.log(`Received chat start event for message ID ${id}`);
......@@ -489,7 +514,6 @@
// Start monitoring and playing audio for the message ID
monitorAndPlayAudio(id, audioAbortController.signal);
}
}
};
const chatEventHandler = async (e) => {
......@@ -499,7 +523,6 @@
// "content" here is a sentence from the assistant,
// there will be many sentences for the same "id"
if ($config.audio.tts.engine !== '') {
if (currentMessageId === id) {
console.log(`Received chat event for message ID ${id}: ${content}`);
......@@ -517,7 +540,6 @@
console.error('Failed to fetch or play audio:', error);
}
}
}
};
const chatFinishHandler = async (e) => {
......@@ -525,12 +547,7 @@
// "content" here is the entire message from the assistant
chatStreaming = false;
if ($config.audio.tts.engine !== '') {
finishedMessages[id] = true;
} else {
speakSpeechSynthesisHandler(content);
}
};
eventTarget.addEventListener('chat:start', chatStartHandler);
......@@ -561,7 +578,20 @@
<div class="max-w-lg w-full h-screen max-h-[100dvh] flex flex-col justify-between p-3 md:p-6">
{#if camera}
<div class="flex justify-center items-center w-full h-20 min-h-20">
{#if loading}
{#if emoji}
<div
class=" transition-all rounded-full"
style="font-size:{rmsLevel * 100 > 4
? '4.5'
: rmsLevel * 100 > 2
? '4.25'
: rmsLevel * 100 > 1
? '3.75'
: '3.5'}rem;width: 100%; text-align:center;"
>
{emoji}
</div>
{:else if loading}
<svg
class="size-12 text-gray-900 dark:text-gray-400"
viewBox="0 0 24 24"
......@@ -598,19 +628,6 @@
r="3"
/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
>
{:else if emoji}
<div
class=" transition-all rounded-full"
style="font-size:{rmsLevel * 100 > 4
? '4.5'
: rmsLevel * 100 > 2
? '4.25'
: rmsLevel * 100 > 1
? '3.75'
: '3.5'}rem;width: 100%; text-align:center;"
>
{emoji}
</div>
{:else}
<div
class=" {rmsLevel * 100 > 4
......@@ -628,7 +645,20 @@
<div class="flex justify-center items-center flex-1 h-full w-full max-h-full">
{#if !camera}
{#if loading}
{#if emoji}
<div
class=" transition-all rounded-full"
style="font-size:{rmsLevel * 100 > 4
? '13'
: rmsLevel * 100 > 2
? '12'
: rmsLevel * 100 > 1
? '11.5'
: '11'}rem;width:100%;text-align:center;"
>
{emoji}
</div>
{:else if loading}
<svg
class="size-44 text-gray-900 dark:text-gray-400"
viewBox="0 0 24 24"
......@@ -665,19 +695,6 @@
r="3"
/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
>
{:else if emoji}
<div
class=" transition-all rounded-full"
style="font-size:{rmsLevel * 100 > 4
? '13'
: rmsLevel * 100 > 2
? '12'
: rmsLevel * 100 > 1
? '11.5'
: '11'}rem;width:100%;text-align:center;"
>
{emoji}
</div>
{:else}
<div
class=" {rmsLevel * 100 > 4
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment