CallOverlay.svelte 21.9 KB
Newer Older
Timothy J. Baek's avatar
Timothy J. Baek committed
1
<script lang="ts">
Timothy J. Baek's avatar
Timothy J. Baek committed
2
	import { config, settings, showCallOverlay } from '$lib/stores';
Timothy J. Baek's avatar
Timothy J. Baek committed
3
4
	import { onMount, tick, getContext } from 'svelte';

Timothy J. Baek's avatar
Timothy J. Baek committed
5
6
7
8
9
10
	import {
		blobToFile,
		calculateSHA256,
		extractSentencesForAudio,
		findWordIndices
	} from '$lib/utils';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
11
	import { generateEmoji } from '$lib/apis';
Timothy J. Baek's avatar
Timothy J. Baek committed
12
	import { synthesizeOpenAISpeech, transcribeAudio } from '$lib/apis/audio';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
13

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
14
	import { toast } from 'svelte-sonner';
Timothy J. Baek's avatar
Timothy J. Baek committed
15

Timothy J. Baek's avatar
Timothy J. Baek committed
16
	import Tooltip from '$lib/components/common/Tooltip.svelte';
Timothy J. Baek's avatar
Timothy J. Baek committed
17
	import VideoInputMenu from './CallOverlay/VideoInputMenu.svelte';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
18

Timothy J. Baek's avatar
Timothy J. Baek committed
19
20
	const i18n = getContext('i18n');

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
21
	export let eventTarget: EventTarget;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
22

23
	export let submitPrompt: Function;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
24
25
	export let stopResponse: Function;

Timothy J. Baek's avatar
Timothy J. Baek committed
26
	export let files;
27

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
28
29
30
	export let chatId;
	export let modelId;

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
31
32
	let loading = false;
	let confirmed = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
33
	let interrupted = false;
Timothy J. Baek's avatar
Timothy J. Baek committed
34

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
35
36
	let emoji = null;

Timothy J. Baek's avatar
Timothy J. Baek committed
37
38
39
	let camera = false;
	let cameraStream = null;

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
40
	let chatStreaming = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
41

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
42
43
	let rmsLevel = 0;
	let hasStartedSpeaking = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
44
45
46
	let mediaRecorder;
	let audioChunks = [];

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
	let videoInputDevices = [];
	let selectedVideoInputDeviceId = null;

	const getVideoInputDevices = async () => {
		const devices = await navigator.mediaDevices.enumerateDevices();
		videoInputDevices = devices.filter((device) => device.kind === 'videoinput');

		if (!!navigator.mediaDevices.getDisplayMedia) {
			videoInputDevices = [
				...videoInputDevices,
				{
					deviceId: 'screen',
					label: 'Screen Share'
				}
			];
		}

		console.log(videoInputDevices);
		if (selectedVideoInputDeviceId === null && videoInputDevices.length > 0) {
			selectedVideoInputDeviceId = videoInputDevices[0].deviceId;
		}
	};

	const startCamera = async () => {
		await getVideoInputDevices();

		if (cameraStream === null) {
			camera = true;
			await tick();
			try {
				await startVideoStream();
			} catch (err) {
				console.error('Error accessing webcam: ', err);
			}
		}
	};

	const startVideoStream = async () => {
		const video = document.getElementById('camera-feed');
		if (video) {
			if (selectedVideoInputDeviceId === 'screen') {
				cameraStream = await navigator.mediaDevices.getDisplayMedia({
					video: {
						cursor: 'always'
					},
					audio: false
				});
			} else {
				cameraStream = await navigator.mediaDevices.getUserMedia({
					video: {
						deviceId: selectedVideoInputDeviceId ? { exact: selectedVideoInputDeviceId } : undefined
					}
				});
			}

			if (cameraStream) {
				await getVideoInputDevices();
				video.srcObject = cameraStream;
				await video.play();
			}
		}
	};

	const stopVideoStream = async () => {
		if (cameraStream) {
			const tracks = cameraStream.getTracks();
			tracks.forEach((track) => track.stop());
		}

		cameraStream = null;
	};

	const takeScreenshot = () => {
		const video = document.getElementById('camera-feed');
		const canvas = document.getElementById('camera-canvas');

		if (!canvas) {
			return;
		}

		const context = canvas.getContext('2d');

		// Make the canvas match the video dimensions
		canvas.width = video.videoWidth;
		canvas.height = video.videoHeight;

		// Draw the image from the video onto the canvas
		context.drawImage(video, 0, 0, video.videoWidth, video.videoHeight);

		// Convert the canvas to a data base64 URL and console log it
		const dataURL = canvas.toDataURL('image/png');
		console.log(dataURL);

		return dataURL;
	};

	const stopCamera = async () => {
		await stopVideoStream();
		camera = false;
	};

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
148
149
150
	const MIN_DECIBELS = -45;
	const VISUALIZER_BUFFER_LENGTH = 300;

Timothy J. Baek's avatar
Timothy J. Baek committed
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
	const transcribeHandler = async (audioBlob) => {
		// Create a blob from the audio chunks

		await tick();
		const file = blobToFile(audioBlob, 'recording.wav');

		const res = await transcribeAudio(localStorage.token, file).catch((error) => {
			toast.error(error);
			return null;
		});

		if (res) {
			console.log(res.text);

			if (res.text !== '') {
				const _responses = await submitPrompt(res.text, { _raw: true });
				console.log(_responses);
			}
		}
	};

	const stopRecordingCallback = async (_continue = true) => {
		if ($showCallOverlay) {
			console.log('%c%s', 'color: red; font-size: 20px;', '🚨 stopRecordingCallback 🚨');

			// deep copy the audioChunks array
			const _audioChunks = audioChunks.slice(0);

			audioChunks = [];
			mediaRecorder = false;

Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
182
183
			await tick();

Timothy J. Baek's avatar
Timothy J. Baek committed
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
			if (_continue) {
				startRecording();
			}

			if (confirmed) {
				loading = true;
				emoji = null;

				if (cameraStream) {
					const imageUrl = takeScreenshot();

					files = [
						{
							type: 'image',
							url: imageUrl
						}
					];
				}

				const audioBlob = new Blob(_audioChunks, { type: 'audio/wav' });
				await transcribeHandler(audioBlob);

				confirmed = false;
				loading = false;
			}
		} else {
			audioChunks = [];
			mediaRecorder = false;
		}
	};

	const startRecording = async () => {
		const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
		mediaRecorder = new MediaRecorder(stream);
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
218

Timothy J. Baek's avatar
Timothy J. Baek committed
219
220
221
222
223
		mediaRecorder.onstart = () => {
			console.log('Recording started');
			audioChunks = [];
			analyseAudio(stream);
		};
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
224

Timothy J. Baek's avatar
Timothy J. Baek committed
225
226
227
228
229
		mediaRecorder.ondataavailable = (event) => {
			if (hasStartedSpeaking) {
				audioChunks.push(event.data);
			}
		};
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
230
231
232
233

		mediaRecorder.onstop = (e) => {
			console.log('Recording stopped', e);
			stopRecordingCallback();
Timothy J. Baek's avatar
Timothy J. Baek committed
234
		};
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
235

Timothy J. Baek's avatar
Timothy J. Baek committed
236
237
238
		mediaRecorder.start();
	};

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
	// Function to calculate the RMS level from time domain data
	const calculateRMS = (data: Uint8Array) => {
		let sumSquares = 0;
		for (let i = 0; i < data.length; i++) {
			const normalizedValue = (data[i] - 128) / 128; // Normalize the data
			sumSquares += normalizedValue * normalizedValue;
		}
		return Math.sqrt(sumSquares / data.length);
	};

	const analyseAudio = (stream) => {
		const audioContext = new AudioContext();
		const audioStreamSource = audioContext.createMediaStreamSource(stream);

		const analyser = audioContext.createAnalyser();
		analyser.minDecibels = MIN_DECIBELS;
		audioStreamSource.connect(analyser);

		const bufferLength = analyser.frequencyBinCount;

		const domainData = new Uint8Array(bufferLength);
		const timeDomainData = new Uint8Array(analyser.fftSize);

		let lastSoundTime = Date.now();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
263
		hasStartedSpeaking = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
264

Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
265
266
		console.log('🔊 Sound detection started', lastSoundTime, hasStartedSpeaking);

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
267
268
		const detectSound = () => {
			const processFrame = () => {
Timothy J. Baek's avatar
Timothy J. Baek committed
269
270
271
				if (!mediaRecorder || !$showCallOverlay) {
					return;
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
272

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
273
274
275
276
277
278
279
280
281
				analyser.getByteTimeDomainData(timeDomainData);
				analyser.getByteFrequencyData(domainData);

				// Calculate RMS level from time domain data
				rmsLevel = calculateRMS(timeDomainData);

				// Check if initial speech/noise has started
				const hasSound = domainData.some((value) => value > 0);
				if (hasSound) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
282
283
					// BIG RED TEXT
					console.log('%c%s', 'color: red; font-size: 20px;', '🔊 Sound detected');
Timothy J. Baek's avatar
Timothy J. Baek committed
284
285
286
287
288
289
290

					if (!hasStartedSpeaking) {
						hasStartedSpeaking = true;
						stopAllAudio();
					}

					lastSoundTime = Date.now();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
291
292
293
294
295
296
297
298
				}

				// Start silence detection only after initial speech/noise has been detected
				if (hasStartedSpeaking) {
					if (Date.now() - lastSoundTime > 2000) {
						confirmed = true;

						if (mediaRecorder) {
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
299
							console.log('%c%s', 'color: red; font-size: 20px;', '🔇 Silence detected');
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
300
							mediaRecorder.stop();
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
301
							return;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
302
303
304
305
306
307
308
309
310
						}
					}
				}

				window.requestAnimationFrame(processFrame);
			};

			window.requestAnimationFrame(processFrame);
		};
Timothy J. Baek's avatar
Timothy J. Baek committed
311

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
312
313
314
		detectSound();
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
315
316
317
	let finishedMessages = {};
	let currentMessageId = null;
	let currentUtterance = null;
Timothy J. Baek's avatar
Timothy J. Baek committed
318

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
	const speakSpeechSynthesisHandler = (content) => {
		if ($showCallOverlay) {
			return new Promise((resolve) => {
				let voices = [];
				const getVoicesLoop = setInterval(async () => {
					voices = await speechSynthesis.getVoices();
					if (voices.length > 0) {
						clearInterval(getVoicesLoop);

						const voice =
							voices
								?.filter(
									(v) => v.voiceURI === ($settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice)
								)
								?.at(0) ?? undefined;

						currentUtterance = new SpeechSynthesisUtterance(content);

						if (voice) {
							currentUtterance.voice = voice;
						}

						speechSynthesis.speak(currentUtterance);
						currentUtterance.onend = async (e) => {
							await new Promise((r) => setTimeout(r, 100));
							resolve(e);
						};
					}
				}, 100);
			});
		} else {
			return Promise.resolve();
		}
	};

	const playAudio = (audio) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
355
		if ($showCallOverlay) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
356
			return new Promise((resolve) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
357
				const audioElement = document.getElementById('audioElement');
358

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
359
				if (audioElement) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
360
					audioElement.src = audio.src;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
361
					audioElement.muted = true;
Timothy J. Baek's avatar
Timothy J. Baek committed
362

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
363
364
365
366
367
368
					audioElement
						.play()
						.then(() => {
							audioElement.muted = false;
						})
						.catch((error) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
369
							console.error(error);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
370
						});
371

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
372
					audioElement.onended = async (e) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
373
374
						await new Promise((r) => setTimeout(r, 100));
						resolve(e);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
375
376
					};
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
377
378
379
380
			});
		} else {
			return Promise.resolve();
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
381
382
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
383
384
	const stopAllAudio = async () => {
		interrupted = true;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
385

Timothy J. Baek's avatar
Timothy J. Baek committed
386
387
		if (chatStreaming) {
			stopResponse();
Timothy J. Baek's avatar
Timothy J. Baek committed
388
389
		}

Timothy J. Baek's avatar
Timothy J. Baek committed
390
391
392
393
		if (currentUtterance) {
			speechSynthesis.cancel();
			currentUtterance = null;
		}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
394

Timothy J. Baek's avatar
Timothy J. Baek committed
395
396
		const audioElement = document.getElementById('audioElement');
		if (audioElement) {
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
397
			audioElement.muted = true;
Timothy J. Baek's avatar
Timothy J. Baek committed
398
399
400
401
			audioElement.pause();
			audioElement.currentTime = 0;
		}
	};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
402

Timothy J. Baek's avatar
Timothy J. Baek committed
403
	let audioAbortController = new AbortController();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
404

Timothy J. Baek's avatar
Timothy J. Baek committed
405
406
407
408
409
410
411
412
413
414
415
416
417
	// Audio cache map where key is the content and value is the Audio object.
	const audioCache = new Map();
	const fetchAudio = async (content) => {
		if (!audioCache.has(content)) {
			try {
				const res = await synthesizeOpenAISpeech(
					localStorage.token,
					$settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice,
					content
				).catch((error) => {
					console.error(error);
					return null;
				});
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
418

Timothy J. Baek's avatar
Timothy J. Baek committed
419
420
421
422
				if (res) {
					const blob = await res.blob();
					const blobUrl = URL.createObjectURL(blob);
					audioCache.set(content, new Audio(blobUrl));
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
423
				}
Timothy J. Baek's avatar
Timothy J. Baek committed
424
425
			} catch (error) {
				console.error('Error synthesizing speech:', error);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
426
427
			}
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
428
		return audioCache.get(content);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
429
	};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
430

Timothy J. Baek's avatar
Timothy J. Baek committed
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
	let messages = {};

	const monitorAndPlayAudio = async (id, signal) => {
		while (!signal.aborted) {
			if (messages[id] && messages[id].length > 0) {
				// Retrieve the next content string from the queue
				const content = messages[id].shift(); // Dequeues the content for playing

				if (audioCache.has(content)) {
					// If content is available in the cache, play it
					try {
						console.log(
							'%c%s',
							'color: red; font-size: 20px;',
							`Playing audio for content: ${content}`
						);

						const audio = audioCache.get(content);
						await playAudio(audio); // Here ensure that playAudio is indeed correct method to execute
						console.log(`Played audio for content: ${content}`);
						await new Promise((resolve) => setTimeout(resolve, 200)); // Wait before retrying to reduce tight loop
					} catch (error) {
						console.error('Error playing audio:', error);
					}
				} else {
					// If not available in the cache, push it back to the queue and delay
					messages[id].unshift(content); // Re-queue the content at the start
					console.log(`Audio for "${content}" not yet available in the cache, re-queued...`);
					await new Promise((resolve) => setTimeout(resolve, 200)); // Wait before retrying to reduce tight loop
Timothy J. Baek's avatar
Timothy J. Baek committed
460
				}
Timothy J. Baek's avatar
Timothy J. Baek committed
461
462
463
464
465
466
			} else if (finishedMessages[id] && messages[id] && messages[id].length === 0) {
				// If the message is finished and there are no more messages to process, break the loop
				break;
			} else {
				// No messages to process, sleep for a bit
				await new Promise((resolve) => setTimeout(resolve, 200));
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
467
468
			}
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
469
		console.log(`Audio monitoring and playing stopped for message ID ${id}`);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
470
471
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
472
473
	onMount(async () => {
		startRecording();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
474

Timothy J. Baek's avatar
Timothy J. Baek committed
475
476
		const chatStartHandler = async (e) => {
			const { id } = e.detail;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
477

Timothy J. Baek's avatar
Timothy J. Baek committed
478
			chatStreaming = true;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
479

Timothy J. Baek's avatar
Timothy J. Baek committed
480
481
482
483
			if ($config.audio.tts.engine !== '') {
				// set currentMessageId to id
				if (currentMessageId !== id) {
					console.log(`Received chat start event for message ID ${id}`);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
484

Timothy J. Baek's avatar
Timothy J. Baek committed
485
486
487
488
489
					currentMessageId = id;
					if (audioAbortController) {
						audioAbortController.abort();
					}
					audioAbortController = new AbortController();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
490

Timothy J. Baek's avatar
Timothy J. Baek committed
491
492
493
					// Start monitoring and playing audio for the message ID
					monitorAndPlayAudio(id, audioAbortController.signal);
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
494
			}
Timothy J. Baek's avatar
Timothy J. Baek committed
495
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
496

Timothy J. Baek's avatar
Timothy J. Baek committed
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
		const chatEventHandler = async (e) => {
			const { id, content } = e.detail;
			// "id" here is message id
			// if "id" is not the same as "currentMessageId" then do not process
			// "content" here is a sentence from the assistant,
			// there will be many sentences for the same "id"

			if ($config.audio.tts.engine !== '') {
				if (currentMessageId === id) {
					console.log(`Received chat event for message ID ${id}: ${content}`);

					try {
						if (messages[id] === undefined) {
							messages[id] = [content];
						} else {
							messages[id].push(content);
						}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
514

Timothy J. Baek's avatar
Timothy J. Baek committed
515
						console.log(content);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
516

Timothy J. Baek's avatar
Timothy J. Baek committed
517
518
519
						fetchAudio(content);
					} catch (error) {
						console.error('Failed to fetch or play audio:', error);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
520
521
					}
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
522
			}
Timothy J. Baek's avatar
Timothy J. Baek committed
523
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
524

Timothy J. Baek's avatar
Timothy J. Baek committed
525
526
527
		const chatFinishHandler = async (e) => {
			const { id, content } = e.detail;
			// "content" here is the entire message from the assistant
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
528

Timothy J. Baek's avatar
Timothy J. Baek committed
529
			chatStreaming = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
530

Timothy J. Baek's avatar
Timothy J. Baek committed
531
532
533
534
			if ($config.audio.tts.engine !== '') {
				finishedMessages[id] = true;
			} else {
				speakSpeechSynthesisHandler(content);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
535
			}
Timothy J. Baek's avatar
Timothy J. Baek committed
536
537
538
539
540
541
542
543
544
545
546
		};

		eventTarget.addEventListener('chat:start', chatStartHandler);
		eventTarget.addEventListener('chat', chatEventHandler);
		eventTarget.addEventListener('chat:finish', chatFinishHandler);

		return async () => {
			eventTarget.removeEventListener('chat:start', chatStartHandler);
			eventTarget.removeEventListener('chat', chatEventHandler);
			eventTarget.removeEventListener('chat:finish', chatFinishHandler);

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
547
548
			audioAbortController.abort();
			await tick();
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
549

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
550
551
			await stopAllAudio();

Timothy J. Baek's avatar
Timothy J. Baek committed
552
553
554
			await stopRecordingCallback(false);
			await stopCamera();
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
555
	});
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
556
557
558
</script>

{#if $showCallOverlay}
Timothy J. Baek's avatar
Timothy J. Baek committed
559
	<div class=" absolute w-full h-screen max-h-[100dvh] flex z-[999] overflow-hidden">
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
560
		<div
Timothy J. Baek's avatar
Timothy J. Baek committed
561
			class="absolute w-full h-screen max-h-[100dvh] bg-white text-gray-700 dark:bg-black dark:text-gray-300 flex justify-center"
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
562
		>
Timothy J. Baek's avatar
Timothy J. Baek committed
563
			<div class="max-w-lg w-full h-screen max-h-[100dvh] flex flex-col justify-between p-3 md:p-6">
Timothy J. Baek's avatar
Timothy J. Baek committed
564
				{#if camera}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
565
					<div class="flex justify-center items-center w-full h-20 min-h-20">
Timothy J. Baek's avatar
Timothy J. Baek committed
566
567
568
569
570
571
572
573
574
						{#if loading}
							<svg
								class="size-12 text-gray-900 dark:text-gray-400"
								viewBox="0 0 24 24"
								fill="currentColor"
								xmlns="http://www.w3.org/2000/svg"
								><style>
									.spinner_qM83 {
										animation: spinner_8HQG 1.05s infinite;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
575
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
576
577
									.spinner_oXPr {
										animation-delay: 0.1s;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
578
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
579
580
									.spinner_ZTLf {
										animation-delay: 0.2s;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
581
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
									@keyframes spinner_8HQG {
										0%,
										57.14% {
											animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
											transform: translate(0);
										}
										28.57% {
											animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
											transform: translateY(-6px);
										}
										100% {
											transform: translate(0);
										}
									}
								</style><circle class="spinner_qM83" cx="4" cy="12" r="3" /><circle
									class="spinner_qM83 spinner_oXPr"
									cx="12"
									cy="12"
									r="3"
								/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
							>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
603
604
605
606
607
608
						{:else if emoji}
							<div
								class="  transition-all rounded-full"
								style="font-size:{rmsLevel * 100 > 4
									? '4.5'
									: rmsLevel * 100 > 2
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
609
									? '4.25'
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
610
									: rmsLevel * 100 > 1
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
611
612
									? '3.75'
									: '3.5'}rem;width: 100%; text-align:center;"
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
613
614
615
							>
								{emoji}
							</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
						{:else}
							<div
								class=" {rmsLevel * 100 > 4
									? ' size-[4.5rem]'
									: rmsLevel * 100 > 2
									? ' size-16'
									: rmsLevel * 100 > 1
									? 'size-14'
									: 'size-12'}  transition-all bg-black dark:bg-white rounded-full"
							/>
						{/if}
						<!-- navbar -->
					</div>
				{/if}

Timothy J. Baek's avatar
Timothy J. Baek committed
631
				<div class="flex justify-center items-center flex-1 h-full w-full max-h-full">
Timothy J. Baek's avatar
Timothy J. Baek committed
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
					{#if !camera}
						{#if loading}
							<svg
								class="size-44 text-gray-900 dark:text-gray-400"
								viewBox="0 0 24 24"
								fill="currentColor"
								xmlns="http://www.w3.org/2000/svg"
								><style>
									.spinner_qM83 {
										animation: spinner_8HQG 1.05s infinite;
									}
									.spinner_oXPr {
										animation-delay: 0.1s;
									}
									.spinner_ZTLf {
										animation-delay: 0.2s;
									}
									@keyframes spinner_8HQG {
										0%,
										57.14% {
											animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
											transform: translate(0);
										}
										28.57% {
											animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
											transform: translateY(-6px);
										}
										100% {
											transform: translate(0);
										}
									}
								</style><circle class="spinner_qM83" cx="4" cy="12" r="3" /><circle
									class="spinner_qM83 spinner_oXPr"
									cx="12"
									cy="12"
									r="3"
								/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
							>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
670
671
672
673
674
675
676
677
678
679
680
681
682
						{:else if emoji}
							<div
								class="  transition-all rounded-full"
								style="font-size:{rmsLevel * 100 > 4
									? '13'
									: rmsLevel * 100 > 2
									? '12'
									: rmsLevel * 100 > 1
									? '11.5'
									: '11'}rem;width:100%;text-align:center;"
							>
								{emoji}
							</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
683
684
685
686
687
688
689
690
691
692
693
						{:else}
							<div
								class=" {rmsLevel * 100 > 4
									? ' size-52'
									: rmsLevel * 100 > 2
									? 'size-48'
									: rmsLevel * 100 > 1
									? 'size-[11.5rem]'
									: 'size-44'}  transition-all bg-black dark:bg-white rounded-full"
							/>
						{/if}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
694
					{:else}
Timothy J. Baek's avatar
Timothy J. Baek committed
695
696
697
						<div
							class="relative flex video-container w-full max-h-full pt-2 pb-4 md:py-6 px-2 h-full"
						>
Timothy J. Baek's avatar
Timothy J. Baek committed
698
699
700
							<video
								id="camera-feed"
								autoplay
Timothy J. Baek's avatar
Timothy J. Baek committed
701
								class="rounded-2xl h-full min-w-full object-cover object-center"
Timothy J. Baek's avatar
Timothy J. Baek committed
702
								playsinline
Timothy J. Baek's avatar
Timothy J. Baek committed
703
704
705
706
							/>

							<canvas id="camera-canvas" style="display:none;" />

Timothy J. Baek's avatar
Timothy J. Baek committed
707
							<div class=" absolute top-4 md:top-8 left-4">
Timothy J. Baek's avatar
Timothy J. Baek committed
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
								<button
									type="button"
									class="p-1.5 text-white cursor-pointer backdrop-blur-xl bg-black/10 rounded-full"
									on:click={() => {
										stopCamera();
									}}
								>
									<svg
										xmlns="http://www.w3.org/2000/svg"
										viewBox="0 0 16 16"
										fill="currentColor"
										class="size-6"
									>
										<path
											d="M5.28 4.22a.75.75 0 0 0-1.06 1.06L6.94 8l-2.72 2.72a.75.75 0 1 0 1.06 1.06L8 9.06l2.72 2.72a.75.75 0 1 0 1.06-1.06L9.06 8l2.72-2.72a.75.75 0 0 0-1.06-1.06L8 6.94 5.28 4.22Z"
										/>
									</svg>
								</button>
							</div>
						</div>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
728
					{/if}
Timothy J. Baek's avatar
Timothy J. Baek committed
729
730
				</div>

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
731
732
				<div class="flex justify-between items-center pb-2 w-full">
					<div>
Timothy J. Baek's avatar
Timothy J. Baek committed
733
734
735
736
737
738
739
740
						{#if camera}
							<VideoInputMenu
								devices={videoInputDevices}
								on:change={async (e) => {
									console.log(e.detail);
									selectedVideoInputDeviceId = e.detail;
									await stopVideoStream();
									await startVideoStream();
Timothy J. Baek's avatar
Timothy J. Baek committed
741
742
								}}
							>
Timothy J. Baek's avatar
Timothy J. Baek committed
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
								<button class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900" type="button">
									<svg
										xmlns="http://www.w3.org/2000/svg"
										viewBox="0 0 20 20"
										fill="currentColor"
										class="size-5"
									>
										<path
											fill-rule="evenodd"
											d="M15.312 11.424a5.5 5.5 0 0 1-9.201 2.466l-.312-.311h2.433a.75.75 0 0 0 0-1.5H3.989a.75.75 0 0 0-.75.75v4.242a.75.75 0 0 0 1.5 0v-2.43l.31.31a7 7 0 0 0 11.712-3.138.75.75 0 0 0-1.449-.39Zm1.23-3.723a.75.75 0 0 0 .219-.53V2.929a.75.75 0 0 0-1.5 0V5.36l-.31-.31A7 7 0 0 0 3.239 8.188a.75.75 0 1 0 1.448.389A5.5 5.5 0 0 1 13.89 6.11l.311.31h-2.432a.75.75 0 0 0 0 1.5h4.243a.75.75 0 0 0 .53-.219Z"
											clip-rule="evenodd"
										/>
									</svg>
								</button>
							</VideoInputMenu>
						{:else}
Timothy J. Baek's avatar
Timothy J. Baek committed
759
							<Tooltip content={$i18n.t('Camera')}>
Timothy J. Baek's avatar
Timothy J. Baek committed
760
761
762
								<button
									class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
									type="button"
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
763
764
									on:click={async () => {
										await navigator.mediaDevices.getUserMedia({ video: true });
Timothy J. Baek's avatar
Timothy J. Baek committed
765
766
										startCamera();
									}}
Timothy J. Baek's avatar
Timothy J. Baek committed
767
								>
Timothy J. Baek's avatar
Timothy J. Baek committed
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
									<svg
										xmlns="http://www.w3.org/2000/svg"
										fill="none"
										viewBox="0 0 24 24"
										stroke-width="1.5"
										stroke="currentColor"
										class="size-5"
									>
										<path
											stroke-linecap="round"
											stroke-linejoin="round"
											d="M6.827 6.175A2.31 2.31 0 0 1 5.186 7.23c-.38.054-.757.112-1.134.175C2.999 7.58 2.25 8.507 2.25 9.574V18a2.25 2.25 0 0 0 2.25 2.25h15A2.25 2.25 0 0 0 21.75 18V9.574c0-1.067-.75-1.994-1.802-2.169a47.865 47.865 0 0 0-1.134-.175 2.31 2.31 0 0 1-1.64-1.055l-.822-1.316a2.192 2.192 0 0 0-1.736-1.039 48.774 48.774 0 0 0-5.232 0 2.192 2.192 0 0 0-1.736 1.039l-.821 1.316Z"
										/>
										<path
											stroke-linecap="round"
											stroke-linejoin="round"
											d="M16.5 12.75a4.5 4.5 0 1 1-9 0 4.5 4.5 0 0 1 9 0ZM18.75 10.5h.008v.008h-.008V10.5Z"
										/>
									</svg>
								</button>
							</Tooltip>
						{/if}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
790
					</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
791

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
792
					<div>
793
						<button type="button">
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
794
795
							<div class=" line-clamp-1 text-sm font-medium">
								{#if loading}
Karl Lee's avatar
Karl Lee committed
796
									{$i18n.t('Thinking...')}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
797
								{:else}
Karl Lee's avatar
Karl Lee committed
798
									{$i18n.t('Listening...')}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
799
800
801
802
803
804
805
806
807
808
809
810
								{/if}
							</div>
						</button>
					</div>

					<div>
						<button
							class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
							on:click={async () => {
								showCallOverlay.set(false);
							}}
							type="button"
Timothy J. Baek's avatar
Timothy J. Baek committed
811
						>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
812
813
814
815
816
817
818
819
820
821
822
823
							<svg
								xmlns="http://www.w3.org/2000/svg"
								viewBox="0 0 20 20"
								fill="currentColor"
								class="size-5"
							>
								<path
									d="M6.28 5.22a.75.75 0 0 0-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 1 0 1.06 1.06L10 11.06l3.72 3.72a.75.75 0 1 0 1.06-1.06L11.06 10l3.72-3.72a.75.75 0 0 0-1.06-1.06L10 8.94 6.28 5.22Z"
								/>
							</svg>
						</button>
					</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
824
825
826
827
				</div>
			</div>
		</div>
	</div>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
828
{/if}