CallOverlay.svelte 20.2 KB
Newer Older
Timothy J. Baek's avatar
Timothy J. Baek committed
1
<script lang="ts">
Timothy J. Baek's avatar
Timothy J. Baek committed
2
	import { config, settings, showCallOverlay } from '$lib/stores';
Timothy J. Baek's avatar
Timothy J. Baek committed
3
4
	import { onMount, tick, getContext } from 'svelte';

Timothy J. Baek's avatar
Timothy J. Baek committed
5
	import { blobToFile, calculateSHA256, extractSentences, findWordIndices } from '$lib/utils';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
6
	import { generateEmoji } from '$lib/apis';
Timothy J. Baek's avatar
Timothy J. Baek committed
7
	import { synthesizeOpenAISpeech, transcribeAudio } from '$lib/apis/audio';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
8

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
9
	import { toast } from 'svelte-sonner';
Timothy J. Baek's avatar
Timothy J. Baek committed
10

Timothy J. Baek's avatar
Timothy J. Baek committed
11
	import Tooltip from '$lib/components/common/Tooltip.svelte';
Timothy J. Baek's avatar
Timothy J. Baek committed
12
	import VideoInputMenu from './CallOverlay/VideoInputMenu.svelte';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
13

Timothy J. Baek's avatar
Timothy J. Baek committed
14
15
	const i18n = getContext('i18n');

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
16
	export let eventTarget: EventTarget;
17
	export let submitPrompt: Function;
Timothy J. Baek's avatar
Timothy J. Baek committed
18
	export let files;
19

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
20
21
22
23
24
	export let chatId;
	export let modelId;

	let message = '';

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
25
26
	let loading = false;
	let confirmed = false;
Timothy J. Baek's avatar
Timothy J. Baek committed
27

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
28
29
	let emoji = null;

Timothy J. Baek's avatar
Timothy J. Baek committed
30
31
32
	let camera = false;
	let cameraStream = null;

33
	let assistantSpeaking = false;
Timothy J. Baek's avatar
Timothy J. Baek committed
34
35
	let assistantAudio = {};
	let assistantAudioIdx = null;
36

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
37
	let rmsLevel = 0;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
38
	let hasStartedSpeaking = false;
Timothy J. Baek's avatar
Timothy J. Baek committed
39

Timothy J. Baek's avatar
Timothy J. Baek committed
40
	let currentUtterance = null;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

	let mediaRecorder;
	let audioChunks = [];

	const MIN_DECIBELS = -45;
	const VISUALIZER_BUFFER_LENGTH = 300;

	// Function to calculate the RMS level from time domain data
	const calculateRMS = (data: Uint8Array) => {
		let sumSquares = 0;
		for (let i = 0; i < data.length; i++) {
			const normalizedValue = (data[i] - 128) / 128; // Normalize the data
			sumSquares += normalizedValue * normalizedValue;
		}
		return Math.sqrt(sumSquares / data.length);
	};

	const normalizeRMS = (rms) => {
		rms = rms * 10;
		const exp = 1.5; // Adjust exponent value; values greater than 1 expand larger numbers more and compress smaller numbers more
		const scaledRMS = Math.pow(rms, exp);

		// Scale between 0.01 (1%) and 1.0 (100%)
		return Math.min(1.0, Math.max(0.01, scaledRMS));
	};

	const analyseAudio = (stream) => {
		const audioContext = new AudioContext();
		const audioStreamSource = audioContext.createMediaStreamSource(stream);

		const analyser = audioContext.createAnalyser();
		analyser.minDecibels = MIN_DECIBELS;
		audioStreamSource.connect(analyser);

		const bufferLength = analyser.frequencyBinCount;

		const domainData = new Uint8Array(bufferLength);
		const timeDomainData = new Uint8Array(analyser.fftSize);

		let lastSoundTime = Date.now();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
81
		hasStartedSpeaking = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
82
83
84

		const detectSound = () => {
			const processFrame = () => {
Timothy J. Baek's avatar
Timothy J. Baek committed
85
86
87
88
89
90
91
				if (!mediaRecorder || !$showCallOverlay) {
					if (mediaRecorder) {
						mediaRecorder.stop();
					}

					return;
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
92
93
94
95
96
97
98
99
100
				analyser.getByteTimeDomainData(timeDomainData);
				analyser.getByteFrequencyData(domainData);

				// Calculate RMS level from time domain data
				rmsLevel = calculateRMS(timeDomainData);

				// Check if initial speech/noise has started
				const hasSound = domainData.some((value) => value > 0);
				if (hasSound) {
Timothy J. Baek's avatar
Timothy J. Baek committed
101
					stopAllAudio();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
					hasStartedSpeaking = true;
					lastSoundTime = Date.now();
				}

				// Start silence detection only after initial speech/noise has been detected
				if (hasStartedSpeaking) {
					if (Date.now() - lastSoundTime > 2000) {
						confirmed = true;

						if (mediaRecorder) {
							mediaRecorder.stop();
						}
					}
				}

				window.requestAnimationFrame(processFrame);
			};

			window.requestAnimationFrame(processFrame);
		};
Timothy J. Baek's avatar
Timothy J. Baek committed
122

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
123
124
125
		detectSound();
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
126
127
128
129
130
131
132
133
134
	const stopAllAudio = () => {
		if (currentUtterance) {
			speechSynthesis.cancel();
			currentUtterance = null;
		}
		if (assistantAudio[assistantAudioIdx]) {
			assistantAudio[assistantAudioIdx].pause();
			assistantAudio[assistantAudioIdx].currentTime = 0;
		}
135
136
137

		const audioElement = document.getElementById('audioElement');

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
138
139
140
141
		if (audioElement) {
			audioElement.pause();
			audioElement.currentTime = 0;
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
142
143
144
145
		assistantSpeaking = false;
	};

	const playAudio = (idx) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
146
147
148
149
150
		if ($showCallOverlay) {
			return new Promise((res) => {
				assistantAudioIdx = idx;
				const audioElement = document.getElementById('audioElement');
				const audio = assistantAudio[idx];
151

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
152
153
				if (audioElement) {
					audioElement.src = audio.src; // Assume `assistantAudio` has objects with a `src` property
Timothy J. Baek's avatar
Timothy J. Baek committed
154

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
155
					audioElement.muted = true;
Timothy J. Baek's avatar
Timothy J. Baek committed
156

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
157
158
159
160
161
162
163
164
					audioElement
						.play()
						.then(() => {
							audioElement.muted = false;
						})
						.catch((error) => {
							toast.error(error);
						});
165

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
166
167
					audioElement.onended = async (e) => {
						await new Promise((r) => setTimeout(r, 300));
Timothy J. Baek's avatar
Timothy J. Baek committed
168

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
169
170
171
						if (Object.keys(assistantAudio).length - 1 === idx) {
							assistantSpeaking = false;
						}
Timothy J. Baek's avatar
Timothy J. Baek committed
172

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
173
174
175
						res(e);
					};
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
176
177
178
179
			});
		} else {
			return Promise.resolve();
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
180
181
182
183
184
	};

	const getOpenAISpeech = async (text) => {
		const res = await synthesizeOpenAISpeech(
			localStorage.token,
Timothy J. Baek's avatar
Timothy J. Baek committed
185
			$settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice,
Timothy J. Baek's avatar
Timothy J. Baek committed
186
			text
Timothy J. Baek's avatar
Timothy J. Baek committed
187
188
189
190
191
192
193
194
195
196
197
198
199
200
		).catch((error) => {
			toast.error(error);
			assistantSpeaking = false;
			return null;
		});

		if (res) {
			const blob = await res.blob();
			const blobUrl = URL.createObjectURL(blob);
			const audio = new Audio(blobUrl);
			assistantAudio = audio;
		}
	};

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
201
202
203
204
205
206
207
208
209
210
211
212
	const transcribeHandler = async (audioBlob) => {
		// Create a blob from the audio chunks

		await tick();
		const file = blobToFile(audioBlob, 'recording.wav');

		const res = await transcribeAudio(localStorage.token, file).catch((error) => {
			toast.error(error);
			return null;
		});

		if (res) {
Timothy J. Baek's avatar
Timothy J. Baek committed
213
			console.log(res.text);
214

Timothy J. Baek's avatar
Timothy J. Baek committed
215
			if (res.text !== '') {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
216
				const _responses = await submitPrompt(res.text, { _raw: true });
Timothy J. Baek's avatar
Timothy J. Baek committed
217
				console.log(_responses);
Timothy J. Baek's avatar
Timothy J. Baek committed
218
219
220
			}
		}
	};
221

Timothy J. Baek's avatar
Timothy J. Baek committed
222
223
224
	const assistantSpeakingHandler = async (content) => {
		assistantSpeaking = true;

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
		if (modelId && ($settings?.showEmojiInCall ?? false)) {
			console.log('Generating emoji');
			const res = await generateEmoji(localStorage.token, modelId, content, chatId).catch(
				(error) => {
					console.error(error);
					return null;
				}
			);

			if (res) {
				console.log(res);
				if (/\p{Extended_Pictographic}/u.test(res)) {
					emoji = res.match(/\p{Extended_Pictographic}/gu)[0];
				}
			}
		}

Timothy J. Baek's avatar
Timothy J. Baek committed
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
		if (($config.audio.tts.engine ?? '') == '') {
			let voices = [];
			const getVoicesLoop = setInterval(async () => {
				voices = await speechSynthesis.getVoices();
				if (voices.length > 0) {
					clearInterval(getVoicesLoop);

					const voice =
						voices
							?.filter(
								(v) => v.voiceURI === ($settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice)
							)
							?.at(0) ?? undefined;

					currentUtterance = new SpeechSynthesisUtterance(content);
Timothy J. Baek's avatar
Timothy J. Baek committed
257
258
259
260
261

					if (voice) {
						currentUtterance.voice = voice;
					}

Timothy J. Baek's avatar
Timothy J. Baek committed
262
					speechSynthesis.speak(currentUtterance);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
263
264
265
266

					currentUtterance.onend = async () => {
						assistantSpeaking = false;
					};
Timothy J. Baek's avatar
Timothy J. Baek committed
267
268
269
				}
			}, 100);
		} else if ($config.audio.tts.engine === 'openai') {
Timothy J. Baek's avatar
Timothy J. Baek committed
270
271
272
273
274
275
276
277
278
			console.log('openai');

			const sentences = extractSentences(content).reduce((mergedTexts, currentText) => {
				const lastIndex = mergedTexts.length - 1;
				if (lastIndex >= 0) {
					const previousText = mergedTexts[lastIndex];
					const wordCount = previousText.split(/\s+/).length;
					if (wordCount < 2) {
						mergedTexts[lastIndex] = previousText + ' ' + currentText;
279
					} else {
Timothy J. Baek's avatar
Timothy J. Baek committed
280
						mergedTexts.push(currentText);
281
					}
Timothy J. Baek's avatar
Timothy J. Baek committed
282
283
284
285
286
287
288
289
290
291
292
293
294
				} else {
					mergedTexts.push(currentText);
				}
				return mergedTexts;
			}, []);

			console.log(sentences);

			let lastPlayedAudioPromise = Promise.resolve(); // Initialize a promise that resolves immediately

			for (const [idx, sentence] of sentences.entries()) {
				const res = await synthesizeOpenAISpeech(
					localStorage.token,
Timothy J. Baek's avatar
Timothy J. Baek committed
295
					$settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice,
Timothy J. Baek's avatar
Timothy J. Baek committed
296
					sentence
Timothy J. Baek's avatar
Timothy J. Baek committed
297
298
299
300
301
302
303
304
305
306
307
308
309
				).catch((error) => {
					toast.error(error);

					assistantSpeaking = false;
					return null;
				});

				if (res) {
					const blob = await res.blob();
					const blobUrl = URL.createObjectURL(blob);
					const audio = new Audio(blobUrl);
					assistantAudio[idx] = audio;
					lastPlayedAudioPromise = lastPlayedAudioPromise.then(() => playAudio(idx));
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
310
311
312
313
314
315

					if (idx === sentences.length - 1) {
						lastPlayedAudioPromise.then(() => {
							assistantSpeaking = false;
						});
					}
316
317
				}
			}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
318
319
320
		}
	};

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
321
	const stopRecordingCallback = async (_continue = true) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
322
323
324
		if ($showCallOverlay) {
			if (confirmed) {
				loading = true;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
325
				emoji = null;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
326

Timothy J. Baek's avatar
Timothy J. Baek committed
327
328
329
330
331
332
333
334
335
336
337
				if (cameraStream) {
					const imageUrl = takeScreenshot();

					files = [
						{
							type: 'image',
							url: imageUrl
						}
					];
				}

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
338
339
340
341
342
343
344
345
346
				const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
				await transcribeHandler(audioBlob);

				confirmed = false;
				loading = false;
			}
			audioChunks = [];
			mediaRecorder = false;

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
347
348
349
			if (_continue) {
				startRecording();
			}
Timothy J. Baek's avatar
Timothy J. Baek committed
350
351
352
		} else {
			audioChunks = [];
			mediaRecorder = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
353
354
355
356
357
358
359
360
361
362
363
		}
	};

	const startRecording = async () => {
		const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
		mediaRecorder = new MediaRecorder(stream);
		mediaRecorder.onstart = () => {
			console.log('Recording started');
			audioChunks = [];
			analyseAudio(stream);
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
364
365
366
367
368
		mediaRecorder.ondataavailable = (event) => {
			if (hasStartedSpeaking) {
				audioChunks.push(event.data);
			}
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
369
370
371
		mediaRecorder.onstop = async () => {
			console.log('Recording stopped');

Timothy J. Baek's avatar
Timothy J. Baek committed
372
			await stopRecordingCallback();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
373
374
375
376
		};
		mediaRecorder.start();
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
377
378
379
380
381
382
383
	let videoInputDevices = [];
	let selectedVideoInputDeviceId = null;

	const getVideoInputDevices = async () => {
		const devices = await navigator.mediaDevices.enumerateDevices();
		videoInputDevices = devices.filter((device) => device.kind === 'videoinput');

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
384
385
386
387
388
389
390
391
392
		if (!!navigator.mediaDevices.getDisplayMedia) {
			videoInputDevices = [
				...videoInputDevices,
				{
					deviceId: 'screen',
					label: 'Screen Share'
				}
			];
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
393
394
395
396
397
398
399

		console.log(videoInputDevices);
		if (selectedVideoInputDeviceId === null && videoInputDevices.length > 0) {
			selectedVideoInputDeviceId = videoInputDevices[0].deviceId;
		}
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
400
	const startCamera = async () => {
Timothy J. Baek's avatar
Timothy J. Baek committed
401
402
		await getVideoInputDevices();

Timothy J. Baek's avatar
Timothy J. Baek committed
403
404
405
406
		if (cameraStream === null) {
			camera = true;
			await tick();
			try {
Timothy J. Baek's avatar
Timothy J. Baek committed
407
				await startVideoStream();
Timothy J. Baek's avatar
Timothy J. Baek committed
408
409
410
411
412
413
			} catch (err) {
				console.error('Error accessing webcam: ', err);
			}
		}
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
	const startVideoStream = async () => {
		const video = document.getElementById('camera-feed');
		if (video) {
			if (selectedVideoInputDeviceId === 'screen') {
				cameraStream = await navigator.mediaDevices.getDisplayMedia({
					video: {
						cursor: 'always'
					},
					audio: false
				});
			} else {
				cameraStream = await navigator.mediaDevices.getUserMedia({
					video: {
						deviceId: selectedVideoInputDeviceId ? { exact: selectedVideoInputDeviceId } : undefined
					}
				});
			}

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
432
433
434
435
436
			if (cameraStream) {
				await getVideoInputDevices();
				video.srcObject = cameraStream;
				await video.play();
			}
Timothy J. Baek's avatar
Timothy J. Baek committed
437
438
439
440
441
442
443
444
445
446
447
448
		}
	};

	const stopVideoStream = async () => {
		if (cameraStream) {
			const tracks = cameraStream.getTracks();
			tracks.forEach((track) => track.stop());
		}

		cameraStream = null;
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
449
450
451
452
453
454
455
456
457
	const takeScreenshot = () => {
		const video = document.getElementById('camera-feed');
		const canvas = document.getElementById('camera-canvas');

		if (!canvas) {
			return;
		}

		const context = canvas.getContext('2d');
Timothy J. Baek's avatar
Timothy J. Baek committed
458

Timothy J. Baek's avatar
Timothy J. Baek committed
459
460
461
		// Make the canvas match the video dimensions
		canvas.width = video.videoWidth;
		canvas.height = video.videoHeight;
Timothy J. Baek's avatar
Timothy J. Baek committed
462
463
464

		// Draw the image from the video onto the canvas
		context.drawImage(video, 0, 0, video.videoWidth, video.videoHeight);
Timothy J. Baek's avatar
Timothy J. Baek committed
465
466
467
468
469
470
471
472

		// Convert the canvas to a data base64 URL and console log it
		const dataURL = canvas.toDataURL('image/png');
		console.log(dataURL);

		return dataURL;
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
473
474
	const stopCamera = async () => {
		await stopVideoStream();
Timothy J. Baek's avatar
Timothy J. Baek committed
475
476
477
		camera = false;
	};

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
478
479
	$: if ($showCallOverlay) {
		startRecording();
Timothy J. Baek's avatar
Timothy J. Baek committed
480
481
	} else {
		stopCamera();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
482
483
		stopAllAudio();
		stopRecordingCallback(false);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
484
	}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505

	onMount(() => {
		console.log(eventTarget);

		eventTarget.addEventListener('chat:start', async (e) => {
			console.log('Chat start event:', e.detail);
			message = '';
		});

		eventTarget.addEventListener('chat', async (e) => {
			const { content } = e.detail;

			message += content;
			console.log('Chat event:', message);
		});

		eventTarget.addEventListener('chat:finish', async (e) => {
			console.log('Chat finish event:', e.detail);
			message = '';
		});
	});
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
506
507
508
</script>

{#if $showCallOverlay}
509
	<audio id="audioElement" src="" style="display: none;" />
Timothy J. Baek's avatar
Timothy J. Baek committed
510
	<div class=" absolute w-full h-screen max-h-[100dvh] flex z-[999] overflow-hidden">
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
511
		<div
Timothy J. Baek's avatar
Timothy J. Baek committed
512
			class="absolute w-full h-screen max-h-[100dvh] bg-white text-gray-700 dark:bg-black dark:text-gray-300 flex justify-center"
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
513
		>
Timothy J. Baek's avatar
Timothy J. Baek committed
514
			<div class="max-w-lg w-full h-screen max-h-[100dvh] flex flex-col justify-between p-3 md:p-6">
Timothy J. Baek's avatar
Timothy J. Baek committed
515
				{#if camera}
Timothy J. Baek's avatar
Timothy J. Baek committed
516
					<div class="flex justify-center items-center w-full min-h-20">
Timothy J. Baek's avatar
Timothy J. Baek committed
517
518
519
520
521
522
523
524
525
						{#if loading}
							<svg
								class="size-12 text-gray-900 dark:text-gray-400"
								viewBox="0 0 24 24"
								fill="currentColor"
								xmlns="http://www.w3.org/2000/svg"
								><style>
									.spinner_qM83 {
										animation: spinner_8HQG 1.05s infinite;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
526
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
527
528
									.spinner_oXPr {
										animation-delay: 0.1s;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
529
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
530
531
									.spinner_ZTLf {
										animation-delay: 0.2s;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
532
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
									@keyframes spinner_8HQG {
										0%,
										57.14% {
											animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
											transform: translate(0);
										}
										28.57% {
											animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
											transform: translateY(-6px);
										}
										100% {
											transform: translate(0);
										}
									}
								</style><circle class="spinner_qM83" cx="4" cy="12" r="3" /><circle
									class="spinner_qM83 spinner_oXPr"
									cx="12"
									cy="12"
									r="3"
								/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
							>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
554
555
556
557
558
559
560
561
562
563
564
565
566
						{:else if emoji}
							<div
								class="  transition-all rounded-full"
								style="font-size:{rmsLevel * 100 > 4
									? '4.5'
									: rmsLevel * 100 > 2
									? '4'
									: rmsLevel * 100 > 1
									? '3.5'
									: '3'}rem;width:100%;text-align:center;"
							>
								{emoji}
							</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
						{:else}
							<div
								class=" {rmsLevel * 100 > 4
									? ' size-[4.5rem]'
									: rmsLevel * 100 > 2
									? ' size-16'
									: rmsLevel * 100 > 1
									? 'size-14'
									: 'size-12'}  transition-all bg-black dark:bg-white rounded-full"
							/>
						{/if}
						<!-- navbar -->
					</div>
				{/if}

Timothy J. Baek's avatar
Timothy J. Baek committed
582
				<div class="flex justify-center items-center flex-1 h-full w-full max-h-full">
Timothy J. Baek's avatar
Timothy J. Baek committed
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
					{#if !camera}
						{#if loading}
							<svg
								class="size-44 text-gray-900 dark:text-gray-400"
								viewBox="0 0 24 24"
								fill="currentColor"
								xmlns="http://www.w3.org/2000/svg"
								><style>
									.spinner_qM83 {
										animation: spinner_8HQG 1.05s infinite;
									}
									.spinner_oXPr {
										animation-delay: 0.1s;
									}
									.spinner_ZTLf {
										animation-delay: 0.2s;
									}
									@keyframes spinner_8HQG {
										0%,
										57.14% {
											animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
											transform: translate(0);
										}
										28.57% {
											animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
											transform: translateY(-6px);
										}
										100% {
											transform: translate(0);
										}
									}
								</style><circle class="spinner_qM83" cx="4" cy="12" r="3" /><circle
									class="spinner_qM83 spinner_oXPr"
									cx="12"
									cy="12"
									r="3"
								/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
							>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
621
622
623
624
625
626
627
628
629
630
631
632
633
						{:else if emoji}
							<div
								class="  transition-all rounded-full"
								style="font-size:{rmsLevel * 100 > 4
									? '13'
									: rmsLevel * 100 > 2
									? '12'
									: rmsLevel * 100 > 1
									? '11.5'
									: '11'}rem;width:100%;text-align:center;"
							>
								{emoji}
							</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
634
635
636
637
638
639
640
641
642
643
644
						{:else}
							<div
								class=" {rmsLevel * 100 > 4
									? ' size-52'
									: rmsLevel * 100 > 2
									? 'size-48'
									: rmsLevel * 100 > 1
									? 'size-[11.5rem]'
									: 'size-44'}  transition-all bg-black dark:bg-white rounded-full"
							/>
						{/if}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
645
					{:else}
Timothy J. Baek's avatar
Timothy J. Baek committed
646
647
648
						<div
							class="relative flex video-container w-full max-h-full pt-2 pb-4 md:py-6 px-2 h-full"
						>
Timothy J. Baek's avatar
Timothy J. Baek committed
649
650
651
							<video
								id="camera-feed"
								autoplay
Timothy J. Baek's avatar
Timothy J. Baek committed
652
								class="rounded-2xl h-full min-w-full object-cover object-center"
Timothy J. Baek's avatar
Timothy J. Baek committed
653
								playsinline
Timothy J. Baek's avatar
Timothy J. Baek committed
654
655
656
657
							/>

							<canvas id="camera-canvas" style="display:none;" />

Timothy J. Baek's avatar
Timothy J. Baek committed
658
							<div class=" absolute top-4 md:top-8 left-4">
Timothy J. Baek's avatar
Timothy J. Baek committed
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
								<button
									type="button"
									class="p-1.5 text-white cursor-pointer backdrop-blur-xl bg-black/10 rounded-full"
									on:click={() => {
										stopCamera();
									}}
								>
									<svg
										xmlns="http://www.w3.org/2000/svg"
										viewBox="0 0 16 16"
										fill="currentColor"
										class="size-6"
									>
										<path
											d="M5.28 4.22a.75.75 0 0 0-1.06 1.06L6.94 8l-2.72 2.72a.75.75 0 1 0 1.06 1.06L8 9.06l2.72 2.72a.75.75 0 1 0 1.06-1.06L9.06 8l2.72-2.72a.75.75 0 0 0-1.06-1.06L8 6.94 5.28 4.22Z"
										/>
									</svg>
								</button>
							</div>
						</div>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
679
					{/if}
Timothy J. Baek's avatar
Timothy J. Baek committed
680
681
				</div>

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
682
683
				<div class="flex justify-between items-center pb-2 w-full">
					<div>
Timothy J. Baek's avatar
Timothy J. Baek committed
684
685
686
687
688
689
690
691
						{#if camera}
							<VideoInputMenu
								devices={videoInputDevices}
								on:change={async (e) => {
									console.log(e.detail);
									selectedVideoInputDeviceId = e.detail;
									await stopVideoStream();
									await startVideoStream();
Timothy J. Baek's avatar
Timothy J. Baek committed
692
693
								}}
							>
Timothy J. Baek's avatar
Timothy J. Baek committed
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
								<button class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900" type="button">
									<svg
										xmlns="http://www.w3.org/2000/svg"
										viewBox="0 0 20 20"
										fill="currentColor"
										class="size-5"
									>
										<path
											fill-rule="evenodd"
											d="M15.312 11.424a5.5 5.5 0 0 1-9.201 2.466l-.312-.311h2.433a.75.75 0 0 0 0-1.5H3.989a.75.75 0 0 0-.75.75v4.242a.75.75 0 0 0 1.5 0v-2.43l.31.31a7 7 0 0 0 11.712-3.138.75.75 0 0 0-1.449-.39Zm1.23-3.723a.75.75 0 0 0 .219-.53V2.929a.75.75 0 0 0-1.5 0V5.36l-.31-.31A7 7 0 0 0 3.239 8.188a.75.75 0 1 0 1.448.389A5.5 5.5 0 0 1 13.89 6.11l.311.31h-2.432a.75.75 0 0 0 0 1.5h4.243a.75.75 0 0 0 .53-.219Z"
											clip-rule="evenodd"
										/>
									</svg>
								</button>
							</VideoInputMenu>
						{:else}
Timothy J. Baek's avatar
Timothy J. Baek committed
710
							<Tooltip content={$i18n.t('Camera')}>
Timothy J. Baek's avatar
Timothy J. Baek committed
711
712
713
								<button
									class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
									type="button"
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
714
715
									on:click={async () => {
										await navigator.mediaDevices.getUserMedia({ video: true });
Timothy J. Baek's avatar
Timothy J. Baek committed
716
717
										startCamera();
									}}
Timothy J. Baek's avatar
Timothy J. Baek committed
718
								>
Timothy J. Baek's avatar
Timothy J. Baek committed
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
									<svg
										xmlns="http://www.w3.org/2000/svg"
										fill="none"
										viewBox="0 0 24 24"
										stroke-width="1.5"
										stroke="currentColor"
										class="size-5"
									>
										<path
											stroke-linecap="round"
											stroke-linejoin="round"
											d="M6.827 6.175A2.31 2.31 0 0 1 5.186 7.23c-.38.054-.757.112-1.134.175C2.999 7.58 2.25 8.507 2.25 9.574V18a2.25 2.25 0 0 0 2.25 2.25h15A2.25 2.25 0 0 0 21.75 18V9.574c0-1.067-.75-1.994-1.802-2.169a47.865 47.865 0 0 0-1.134-.175 2.31 2.31 0 0 1-1.64-1.055l-.822-1.316a2.192 2.192 0 0 0-1.736-1.039 48.774 48.774 0 0 0-5.232 0 2.192 2.192 0 0 0-1.736 1.039l-.821 1.316Z"
										/>
										<path
											stroke-linecap="round"
											stroke-linejoin="round"
											d="M16.5 12.75a4.5 4.5 0 1 1-9 0 4.5 4.5 0 0 1 9 0ZM18.75 10.5h.008v.008h-.008V10.5Z"
										/>
									</svg>
								</button>
							</Tooltip>
						{/if}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
741
					</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
742

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
743
					<div>
744
						<button type="button">
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
745
746
							<div class=" line-clamp-1 text-sm font-medium">
								{#if loading}
Karl Lee's avatar
Karl Lee committed
747
									{$i18n.t('Thinking...')}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
748
								{:else}
Karl Lee's avatar
Karl Lee committed
749
									{$i18n.t('Listening...')}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
750
751
752
753
754
755
756
757
758
759
760
761
								{/if}
							</div>
						</button>
					</div>

					<div>
						<button
							class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
							on:click={async () => {
								showCallOverlay.set(false);
							}}
							type="button"
Timothy J. Baek's avatar
Timothy J. Baek committed
762
						>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
763
764
765
766
767
768
769
770
771
772
773
774
							<svg
								xmlns="http://www.w3.org/2000/svg"
								viewBox="0 0 20 20"
								fill="currentColor"
								class="size-5"
							>
								<path
									d="M6.28 5.22a.75.75 0 0 0-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 1 0 1.06 1.06L10 11.06l3.72 3.72a.75.75 0 1 0 1.06-1.06L11.06 10l3.72-3.72a.75.75 0 0 0-1.06-1.06L10 8.94 6.28 5.22Z"
								/>
							</svg>
						</button>
					</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
775
776
777
778
				</div>
			</div>
		</div>
	</div>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
779
{/if}