refac

backend/main.py

@@ -887,7 +887,7 @@ async def generate_emoji(form_data: dict, user=Depends(get_verified_user)):
     model = app.state.MODELS[model_id]
 
     template = '''
-You are a perceptive assistant skilled at interpreting emotions from a provided message. Your task is to reflect the speaker's likely facial expression through a fitting emoji. Prioritize using diverse facial expression emojis to convey the nuanced emotions expressed in the text. Please choose ones that vividly represent the speaker's mood or reaction.
+Your task is to reflect the speaker's likely facial expression through a fitting emoji. Interpret emotions from the message and reflect their facial expression using fitting, diverse emojis (e.g., 😊, 😢, 😡, 😱).
 
 Message: """{{prompt}}"""
 '''

src/lib/components/chat/MessageInput/CallOverlay.svelte

@@ -41,6 +41,7 @@
 	let assistantSentenceIdx = -1;
 
 	let audioQueue = [];
+	let emojiQueue = [];
 
 	$: assistantSentences = extractSentences(assistantMessage).reduce((mergedTexts, currentText) => {
 		const lastIndex = mergedTexts.length - 1;
@@ -65,8 +66,6 @@
 	let mediaRecorder;
 	let audioChunks = [];
 
-	$: console.log('hasStartedSpeaking', hasStartedSpeaking);
-
 	let videoInputDevices = [];
 	let selectedVideoInputDeviceId = null;
 
@@ -274,6 +273,7 @@
 		}
 
 		await tick();
+		emojiQueue = [];
 		audioQueue = [];
 		await tick();
 
@@ -354,6 +354,14 @@
 		console.log('playAudioHandler', audioQueue, assistantSpeaking, audioQueue.length > 0);
 		if (!assistantSpeaking && !interrupted && audioQueue.length > 0) {
 			assistantSpeaking = true;
+
+			if ($settings?.showEmojiInCall ?? false) {
+				if (emojiQueue.length > 0) {
+					emoji = emojiQueue.shift();
+					emojiQueue = emojiQueue;
+				}
+			}
+
 			const audioToPlay = audioQueue.shift(); // Shift the audio out from queue before playing.
 			audioQueue = audioQueue;
 			await playAudio(audioToPlay);
@@ -363,9 +371,39 @@
 
 	const setContentAudio = async (content, idx) => {
 		if (assistantSentenceAudios[idx] === undefined) {
-			console.log('%c%s', 'color: red; font-size: 20px;', content);
+			// Wait for the previous audio to be loaded
+			if (idx > 0) {
+				await new Promise((resolve) => {
+					const check = setInterval(() => {
+						if (
+							assistantSentenceAudios[idx - 1] !== undefined &&
+							assistantSentenceAudios[idx - 1] !== null
+						) {
+							clearInterval(check);
+							resolve();
+						}
+					}, 100);
+				});
+			}
 
 			assistantSentenceAudios[idx] = null;
+
+			if ($settings?.showEmojiInCall ?? false) {
+				const sentenceEmoji = await generateEmoji(localStorage.token, modelId, content);
+
+				if (sentenceEmoji) {
+					// Big red text with content and emoji
+					console.log('%c%s', 'color: blue; font-size: 10px;', `${sentenceEmoji}: ${content}`);
+
+					if (/\p{Extended_Pictographic}/u.test(sentenceEmoji)) {
+						emojiQueue.push(sentenceEmoji.match(/\p{Extended_Pictographic}/gu)[0]);
+						emojiQueue = emojiQueue;
+					}
+				}
+
+				await tick();
+			}
+
 			const res = await synthesizeOpenAISpeech(
 				localStorage.token,
 				$settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice,
@@ -381,6 +419,9 @@
 				const blobUrl = URL.createObjectURL(blob);
 				const audio = new Audio(blobUrl);
 				assistantSentenceAudios[idx] = audio;
+
+				console.log('%c%s', 'color: red; font-size: 20px;', content);
+
 				audioQueue.push(audio);
 				audioQueue = audioQueue;
 			}
@@ -388,9 +429,9 @@
 	};
 
 	const stopRecordingCallback = async (_continue = true) => {
+		if ($showCallOverlay) {
 			console.log('%c%s', 'color: red; font-size: 20px;', '🚨 stopRecordingCallback 🚨');
 
-		if ($showCallOverlay) {
 			// deep copy the audioChunks array
 			const _audioChunks = audioChunks.slice(0);
 
@@ -448,13 +489,31 @@
 		mediaRecorder.start();
 	};
 
-	$: if ($showCallOverlay) {
+	const resetAssistantMessage = async () => {
+		interrupted = false;
+
+		assistantMessage = '';
+		assistantSentenceIdx = -1;
+		assistantSentenceAudios = {}; // Reset audio tracking
+		audioQueue = []; // Clear the audio queue
+		audioQueue = audioQueue;
+
+		emoji = null;
+		emojiQueue = [];
+		emojiQueue = emojiQueue;
+	};
+
+	$: (async () => {
+		if ($showCallOverlay) {
+			await resetAssistantMessage();
+			await tick();
 			startRecording();
 		} else {
 			stopCamera();
 			stopAllAudio();
 			stopRecordingCallback(false);
 		}
+	})();
 
 	$: {
 		if (audioQueue.length > 0 && !assistantSpeaking) {
@@ -463,21 +522,17 @@
 	}
 
 	onMount(() => {
-		console.log(eventTarget);
-
 		eventTarget.addEventListener('chat:start', async (e) => {
+			if ($showCallOverlay) {
 				console.log('Chat start event:', e);
-			interrupted = false;
-
-			assistantMessage = '';
-			assistantSentenceIdx = -1;
-			assistantSentenceAudios = {}; // Reset audio tracking
-			audioQueue = []; // Clear the audio queue
-
+				await resetAssistantMessage();
+				await tick();
 				chatStreaming = true;
+			}
 		});
 
 		eventTarget.addEventListener('chat', async (e) => {
+			if ($showCallOverlay) {
 				const { content } = e.detail;
 				assistantMessage += content;
 				await tick();
@@ -494,9 +549,11 @@
 				}
 
 				chatStreaming = true;
+			}
 		});
 
 		eventTarget.addEventListener('chat:finish', async (e) => {
+			if ($showCallOverlay) {
 				chatStreaming = false;
 				loading = false;
 
@@ -512,10 +569,21 @@
 							}
 						}
 					} else {
-					emoji = generateEmoji(localStorage.token, modelId, assistantMessage);
+						if ($settings?.showEmojiInCall ?? false) {
+							const res = await generateEmoji(localStorage.token, modelId, assistantMessage);
+
+							if (res) {
+								console.log(res);
+								if (/\p{Extended_Pictographic}/u.test(res)) {
+									emoji = res.match(/\p{Extended_Pictographic}/gu)[0];
+								}
+							}
+						}
+
 						speakSpeechSynthesisHandler(assistantMessage);
 					}
 				}
+			}
 		});
 	});
 </script>
@@ -529,7 +597,7 @@
 		>
 			<div class="max-w-lg w-full h-screen max-h-[100dvh] flex flex-col justify-between p-3 md:p-6">
 				{#if camera}
-					<div class="flex justify-center items-center w-full min-h-20">
+					<div class="flex justify-center items-center w-full h-20 min-h-20">
 						{#if loading}
 							<svg
 								class="size-12 text-gray-900 dark:text-gray-400"
@@ -573,10 +641,10 @@
 								style="font-size:{rmsLevel * 100 > 4
 									? '4.5'
 									: rmsLevel * 100 > 2
-									? '4'
+									? '4.25'
 									: rmsLevel * 100 > 1
-									? '3.5'
-									: '3'}rem;width:100%;text-align:center;"
+									? '3.75'
+									: '3.5'}rem;width: 100%; text-align:center;"
 							>
 								{emoji}
 							</div>