CallOverlay.svelte 23.3 KB
Newer Older
Timothy J. Baek's avatar
Timothy J. Baek committed
1
<script lang="ts">
Timothy J. Baek's avatar
Timothy J. Baek committed
2
	import { config, settings, showCallOverlay } from '$lib/stores';
Timothy J. Baek's avatar
Timothy J. Baek committed
3
4
	import { onMount, tick, getContext } from 'svelte';

Timothy J. Baek's avatar
Timothy J. Baek committed
5
6
7
8
9
10
	import {
		blobToFile,
		calculateSHA256,
		extractSentencesForAudio,
		findWordIndices
	} from '$lib/utils';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
11
	import { generateEmoji } from '$lib/apis';
Timothy J. Baek's avatar
Timothy J. Baek committed
12
	import { synthesizeOpenAISpeech, transcribeAudio } from '$lib/apis/audio';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
13

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
14
	import { toast } from 'svelte-sonner';
Timothy J. Baek's avatar
Timothy J. Baek committed
15

Timothy J. Baek's avatar
Timothy J. Baek committed
16
	import Tooltip from '$lib/components/common/Tooltip.svelte';
Timothy J. Baek's avatar
Timothy J. Baek committed
17
	import VideoInputMenu from './CallOverlay/VideoInputMenu.svelte';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
18

Timothy J. Baek's avatar
Timothy J. Baek committed
19
20
	const i18n = getContext('i18n');

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
21
	export let eventTarget: EventTarget;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
22

23
	export let submitPrompt: Function;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
24
25
	export let stopResponse: Function;

Timothy J. Baek's avatar
Timothy J. Baek committed
26
	export let files;
27

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
28
29
30
	export let chatId;
	export let modelId;

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
31
32
	let loading = false;
	let confirmed = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
33
	let interrupted = false;
34
	let assistantSpeaking = false;
Timothy J. Baek's avatar
Timothy J. Baek committed
35

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
36
37
	let emoji = null;

Timothy J. Baek's avatar
Timothy J. Baek committed
38
39
40
	let camera = false;
	let cameraStream = null;

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
41
	let chatStreaming = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
42

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
43
44
	let rmsLevel = 0;
	let hasStartedSpeaking = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
45
46
47
	let mediaRecorder;
	let audioChunks = [];

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
	let videoInputDevices = [];
	let selectedVideoInputDeviceId = null;

	const getVideoInputDevices = async () => {
		const devices = await navigator.mediaDevices.enumerateDevices();
		videoInputDevices = devices.filter((device) => device.kind === 'videoinput');

		if (!!navigator.mediaDevices.getDisplayMedia) {
			videoInputDevices = [
				...videoInputDevices,
				{
					deviceId: 'screen',
					label: 'Screen Share'
				}
			];
		}

		console.log(videoInputDevices);
		if (selectedVideoInputDeviceId === null && videoInputDevices.length > 0) {
			selectedVideoInputDeviceId = videoInputDevices[0].deviceId;
		}
	};

	const startCamera = async () => {
		await getVideoInputDevices();

		if (cameraStream === null) {
			camera = true;
			await tick();
			try {
				await startVideoStream();
			} catch (err) {
				console.error('Error accessing webcam: ', err);
			}
		}
	};

	const startVideoStream = async () => {
		const video = document.getElementById('camera-feed');
		if (video) {
			if (selectedVideoInputDeviceId === 'screen') {
				cameraStream = await navigator.mediaDevices.getDisplayMedia({
					video: {
						cursor: 'always'
					},
					audio: false
				});
			} else {
				cameraStream = await navigator.mediaDevices.getUserMedia({
					video: {
						deviceId: selectedVideoInputDeviceId ? { exact: selectedVideoInputDeviceId } : undefined
					}
				});
			}

			if (cameraStream) {
				await getVideoInputDevices();
				video.srcObject = cameraStream;
				await video.play();
			}
		}
	};

	const stopVideoStream = async () => {
		if (cameraStream) {
			const tracks = cameraStream.getTracks();
			tracks.forEach((track) => track.stop());
		}

		cameraStream = null;
	};

	const takeScreenshot = () => {
		const video = document.getElementById('camera-feed');
		const canvas = document.getElementById('camera-canvas');

		if (!canvas) {
			return;
		}

		const context = canvas.getContext('2d');

		// Make the canvas match the video dimensions
		canvas.width = video.videoWidth;
		canvas.height = video.videoHeight;

		// Draw the image from the video onto the canvas
		context.drawImage(video, 0, 0, video.videoWidth, video.videoHeight);

		// Convert the canvas to a data base64 URL and console log it
		const dataURL = canvas.toDataURL('image/png');
		console.log(dataURL);

		return dataURL;
	};

	const stopCamera = async () => {
		await stopVideoStream();
		camera = false;
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
149
	const MIN_DECIBELS = -55;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
150
151
	const VISUALIZER_BUFFER_LENGTH = 300;

Timothy J. Baek's avatar
Timothy J. Baek committed
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
	const transcribeHandler = async (audioBlob) => {
		// Create a blob from the audio chunks

		await tick();
		const file = blobToFile(audioBlob, 'recording.wav');

		const res = await transcribeAudio(localStorage.token, file).catch((error) => {
			toast.error(error);
			return null;
		});

		if (res) {
			console.log(res.text);

			if (res.text !== '') {
				const _responses = await submitPrompt(res.text, { _raw: true });
				console.log(_responses);
			}
		}
	};

	const stopRecordingCallback = async (_continue = true) => {
		if ($showCallOverlay) {
			console.log('%c%s', 'color: red; font-size: 20px;', '🚨 stopRecordingCallback 🚨');

			// deep copy the audioChunks array
			const _audioChunks = audioChunks.slice(0);

			audioChunks = [];
			mediaRecorder = false;

			if (_continue) {
				startRecording();
			}

			if (confirmed) {
				loading = true;
				emoji = null;

				if (cameraStream) {
					const imageUrl = takeScreenshot();

					files = [
						{
							type: 'image',
							url: imageUrl
						}
					];
				}

				const audioBlob = new Blob(_audioChunks, { type: 'audio/wav' });
				await transcribeHandler(audioBlob);

				confirmed = false;
				loading = false;
			}
		} else {
			audioChunks = [];
			mediaRecorder = false;
		}
	};

	const startRecording = async () => {
		const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
		mediaRecorder = new MediaRecorder(stream);
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
217

Timothy J. Baek's avatar
Timothy J. Baek committed
218
219
220
221
222
		mediaRecorder.onstart = () => {
			console.log('Recording started');
			audioChunks = [];
			analyseAudio(stream);
		};
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
223

Timothy J. Baek's avatar
Timothy J. Baek committed
224
225
226
227
228
		mediaRecorder.ondataavailable = (event) => {
			if (hasStartedSpeaking) {
				audioChunks.push(event.data);
			}
		};
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
229
230
231
232

		mediaRecorder.onstop = (e) => {
			console.log('Recording stopped', e);
			stopRecordingCallback();
Timothy J. Baek's avatar
Timothy J. Baek committed
233
		};
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
234

Timothy J. Baek's avatar
Timothy J. Baek committed
235
236
237
		mediaRecorder.start();
	};

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
	// Function to calculate the RMS level from time domain data
	const calculateRMS = (data: Uint8Array) => {
		let sumSquares = 0;
		for (let i = 0; i < data.length; i++) {
			const normalizedValue = (data[i] - 128) / 128; // Normalize the data
			sumSquares += normalizedValue * normalizedValue;
		}
		return Math.sqrt(sumSquares / data.length);
	};

	const analyseAudio = (stream) => {
		const audioContext = new AudioContext();
		const audioStreamSource = audioContext.createMediaStreamSource(stream);

		const analyser = audioContext.createAnalyser();
		analyser.minDecibels = MIN_DECIBELS;
		audioStreamSource.connect(analyser);

		const bufferLength = analyser.frequencyBinCount;

		const domainData = new Uint8Array(bufferLength);
		const timeDomainData = new Uint8Array(analyser.fftSize);

		let lastSoundTime = Date.now();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
262
		hasStartedSpeaking = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
263

Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
264
265
		console.log('🔊 Sound detection started', lastSoundTime, hasStartedSpeaking);

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
266
267
		const detectSound = () => {
			const processFrame = () => {
Timothy J. Baek's avatar
Timothy J. Baek committed
268
269
270
				if (!mediaRecorder || !$showCallOverlay) {
					return;
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
271

272
				if (assistantSpeaking) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
273
					// Mute the audio if the assistant is speaking
274
					analyser.maxDecibels = 0;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
275
					analyser.minDecibels = -1;
276
277
278
279
280
				} else {
					analyser.minDecibels = MIN_DECIBELS;
					analyser.maxDecibels = -30;
				}

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
281
282
283
284
285
286
287
288
289
				analyser.getByteTimeDomainData(timeDomainData);
				analyser.getByteFrequencyData(domainData);

				// Calculate RMS level from time domain data
				rmsLevel = calculateRMS(timeDomainData);

				// Check if initial speech/noise has started
				const hasSound = domainData.some((value) => value > 0);
				if (hasSound) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
290
291
					// BIG RED TEXT
					console.log('%c%s', 'color: red; font-size: 20px;', '🔊 Sound detected');
Timothy J. Baek's avatar
Timothy J. Baek committed
292
293
294
295
296
297
298

					if (!hasStartedSpeaking) {
						hasStartedSpeaking = true;
						stopAllAudio();
					}

					lastSoundTime = Date.now();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
299
300
301
302
303
304
305
306
				}

				// Start silence detection only after initial speech/noise has been detected
				if (hasStartedSpeaking) {
					if (Date.now() - lastSoundTime > 2000) {
						confirmed = true;

						if (mediaRecorder) {
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
307
							console.log('%c%s', 'color: red; font-size: 20px;', '🔇 Silence detected');
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
308
							mediaRecorder.stop();
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
309
							return;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
310
311
312
313
314
315
316
317
318
						}
					}
				}

				window.requestAnimationFrame(processFrame);
			};

			window.requestAnimationFrame(processFrame);
		};
Timothy J. Baek's avatar
Timothy J. Baek committed
319

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
320
321
322
		detectSound();
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
323
324
325
	let finishedMessages = {};
	let currentMessageId = null;
	let currentUtterance = null;
Timothy J. Baek's avatar
Timothy J. Baek committed
326

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
	const speakSpeechSynthesisHandler = (content) => {
		if ($showCallOverlay) {
			return new Promise((resolve) => {
				let voices = [];
				const getVoicesLoop = setInterval(async () => {
					voices = await speechSynthesis.getVoices();
					if (voices.length > 0) {
						clearInterval(getVoicesLoop);

						const voice =
							voices
								?.filter(
									(v) => v.voiceURI === ($settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice)
								)
								?.at(0) ?? undefined;

						currentUtterance = new SpeechSynthesisUtterance(content);

						if (voice) {
							currentUtterance.voice = voice;
						}

						speechSynthesis.speak(currentUtterance);
						currentUtterance.onend = async (e) => {
Timothy J. Baek's avatar
Timothy J. Baek committed
351
							await new Promise((r) => setTimeout(r, 200));
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
352
353
354
355
356
357
358
359
360
361
362
							resolve(e);
						};
					}
				}, 100);
			});
		} else {
			return Promise.resolve();
		}
	};

	const playAudio = (audio) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
363
		if ($showCallOverlay) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
364
			return new Promise((resolve) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
365
				const audioElement = document.getElementById('audioElement');
366

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
367
				if (audioElement) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
368
					audioElement.src = audio.src;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
369
					audioElement.muted = true;
Timothy J. Baek's avatar
Timothy J. Baek committed
370

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
371
372
373
374
375
376
					audioElement
						.play()
						.then(() => {
							audioElement.muted = false;
						})
						.catch((error) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
377
							console.error(error);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
378
						});
379

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
380
					audioElement.onended = async (e) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
381
382
						await new Promise((r) => setTimeout(r, 100));
						resolve(e);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
383
384
					};
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
385
386
387
388
			});
		} else {
			return Promise.resolve();
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
389
390
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
391
	const stopAllAudio = async () => {
392
		assistantSpeaking = false;
Timothy J. Baek's avatar
Timothy J. Baek committed
393
		interrupted = true;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
394

Timothy J. Baek's avatar
Timothy J. Baek committed
395
396
		if (chatStreaming) {
			stopResponse();
Timothy J. Baek's avatar
Timothy J. Baek committed
397
398
		}

Timothy J. Baek's avatar
Timothy J. Baek committed
399
400
401
402
		if (currentUtterance) {
			speechSynthesis.cancel();
			currentUtterance = null;
		}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
403

Timothy J. Baek's avatar
Timothy J. Baek committed
404
405
		const audioElement = document.getElementById('audioElement');
		if (audioElement) {
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
406
			audioElement.muted = true;
Timothy J. Baek's avatar
Timothy J. Baek committed
407
408
409
410
			audioElement.pause();
			audioElement.currentTime = 0;
		}
	};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
411

Timothy J. Baek's avatar
Timothy J. Baek committed
412
	let audioAbortController = new AbortController();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
413

Timothy J. Baek's avatar
Timothy J. Baek committed
414
415
	// Audio cache map where key is the content and value is the Audio object.
	const audioCache = new Map();
Timothy J. Baek's avatar
Timothy J. Baek committed
416
417
	const emojiCache = new Map();

Timothy J. Baek's avatar
Timothy J. Baek committed
418
419
420
	const fetchAudio = async (content) => {
		if (!audioCache.has(content)) {
			try {
Timothy J. Baek's avatar
Timothy J. Baek committed
421
422
423
424
425
426
427
				// Set the emoji for the content if needed
				if ($settings?.showEmojiInCall ?? false) {
					const emoji = await generateEmoji(localStorage.token, modelId, content, chatId);
					if (emoji) {
						emojiCache.set(content, emoji);
					}
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
428

Timothy J. Baek's avatar
Timothy J. Baek committed
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
				if ($config.audio.tts.engine !== '') {
					const res = await synthesizeOpenAISpeech(
						localStorage.token,
						$settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice,
						content
					).catch((error) => {
						console.error(error);
						return null;
					});

					if (res) {
						const blob = await res.blob();
						const blobUrl = URL.createObjectURL(blob);
						audioCache.set(content, new Audio(blobUrl));
					}
				} else {
					audioCache.set(content, true);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
446
				}
Timothy J. Baek's avatar
Timothy J. Baek committed
447
448
			} catch (error) {
				console.error('Error synthesizing speech:', error);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
449
450
			}
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
451

Timothy J. Baek's avatar
Timothy J. Baek committed
452
		return audioCache.get(content);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
453
	};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
454

Timothy J. Baek's avatar
Timothy J. Baek committed
455
456
457
458
459
460
461
462
463
464
	let messages = {};

	const monitorAndPlayAudio = async (id, signal) => {
		while (!signal.aborted) {
			if (messages[id] && messages[id].length > 0) {
				// Retrieve the next content string from the queue
				const content = messages[id].shift(); // Dequeues the content for playing

				if (audioCache.has(content)) {
					// If content is available in the cache, play it
Timothy J. Baek's avatar
Timothy J. Baek committed
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489

					// Set the emoji for the content if available
					if (($settings?.showEmojiInCall ?? false) && emojiCache.has(content)) {
						emoji = emojiCache.get(content);
					} else {
						emoji = null;
					}

					if ($config.audio.tts.engine !== '') {
						try {
							console.log(
								'%c%s',
								'color: red; font-size: 20px;',
								`Playing audio for content: ${content}`
							);

							const audio = audioCache.get(content);
							await playAudio(audio); // Here ensure that playAudio is indeed correct method to execute
							console.log(`Played audio for content: ${content}`);
							await new Promise((resolve) => setTimeout(resolve, 200)); // Wait before retrying to reduce tight loop
						} catch (error) {
							console.error('Error playing audio:', error);
						}
					} else {
						await speakSpeechSynthesisHandler(content);
Timothy J. Baek's avatar
Timothy J. Baek committed
490
491
492
493
494
495
					}
				} else {
					// If not available in the cache, push it back to the queue and delay
					messages[id].unshift(content); // Re-queue the content at the start
					console.log(`Audio for "${content}" not yet available in the cache, re-queued...`);
					await new Promise((resolve) => setTimeout(resolve, 200)); // Wait before retrying to reduce tight loop
Timothy J. Baek's avatar
Timothy J. Baek committed
496
				}
Timothy J. Baek's avatar
Timothy J. Baek committed
497
498
			} else if (finishedMessages[id] && messages[id] && messages[id].length === 0) {
				// If the message is finished and there are no more messages to process, break the loop
499
				assistantSpeaking = false;
Timothy J. Baek's avatar
Timothy J. Baek committed
500
501
502
503
				break;
			} else {
				// No messages to process, sleep for a bit
				await new Promise((resolve) => setTimeout(resolve, 200));
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
504
505
			}
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
506
		console.log(`Audio monitoring and playing stopped for message ID ${id}`);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
507
508
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
509
510
	onMount(async () => {
		startRecording();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
511

Timothy J. Baek's avatar
Timothy J. Baek committed
512
513
		const chatStartHandler = async (e) => {
			const { id } = e.detail;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
514

Timothy J. Baek's avatar
Timothy J. Baek committed
515
			chatStreaming = true;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
516

Timothy J. Baek's avatar
Timothy J. Baek committed
517
518
			if (currentMessageId !== id) {
				console.log(`Received chat start event for message ID ${id}`);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
519

Timothy J. Baek's avatar
Timothy J. Baek committed
520
521
522
				currentMessageId = id;
				if (audioAbortController) {
					audioAbortController.abort();
Timothy J. Baek's avatar
Timothy J. Baek committed
523
				}
Timothy J. Baek's avatar
Timothy J. Baek committed
524
525
				audioAbortController = new AbortController();

526
				assistantSpeaking = true;
Timothy J. Baek's avatar
Timothy J. Baek committed
527
528
				// Start monitoring and playing audio for the message ID
				monitorAndPlayAudio(id, audioAbortController.signal);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
529
			}
Timothy J. Baek's avatar
Timothy J. Baek committed
530
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
531

Timothy J. Baek's avatar
Timothy J. Baek committed
532
533
534
535
536
537
538
		const chatEventHandler = async (e) => {
			const { id, content } = e.detail;
			// "id" here is message id
			// if "id" is not the same as "currentMessageId" then do not process
			// "content" here is a sentence from the assistant,
			// there will be many sentences for the same "id"

Timothy J. Baek's avatar
Timothy J. Baek committed
539
540
			if (currentMessageId === id) {
				console.log(`Received chat event for message ID ${id}: ${content}`);
Timothy J. Baek's avatar
Timothy J. Baek committed
541

Timothy J. Baek's avatar
Timothy J. Baek committed
542
543
544
545
546
547
				try {
					if (messages[id] === undefined) {
						messages[id] = [content];
					} else {
						messages[id].push(content);
					}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
548

Timothy J. Baek's avatar
Timothy J. Baek committed
549
					console.log(content);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
550

Timothy J. Baek's avatar
Timothy J. Baek committed
551
552
553
					fetchAudio(content);
				} catch (error) {
					console.error('Failed to fetch or play audio:', error);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
554
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
555
			}
Timothy J. Baek's avatar
Timothy J. Baek committed
556
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
557

Timothy J. Baek's avatar
Timothy J. Baek committed
558
559
560
		const chatFinishHandler = async (e) => {
			const { id, content } = e.detail;
			// "content" here is the entire message from the assistant
561
			finishedMessages[id] = true;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
562

Timothy J. Baek's avatar
Timothy J. Baek committed
563
564
565
566
567
568
569
570
571
572
573
574
			chatStreaming = false;
		};

		eventTarget.addEventListener('chat:start', chatStartHandler);
		eventTarget.addEventListener('chat', chatEventHandler);
		eventTarget.addEventListener('chat:finish', chatFinishHandler);

		return async () => {
			eventTarget.removeEventListener('chat:start', chatStartHandler);
			eventTarget.removeEventListener('chat', chatEventHandler);
			eventTarget.removeEventListener('chat:finish', chatFinishHandler);

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
575
576
			audioAbortController.abort();
			await tick();
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
577

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
578
579
			await stopAllAudio();

Timothy J. Baek's avatar
Timothy J. Baek committed
580
581
582
			await stopRecordingCallback(false);
			await stopCamera();
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
583
	});
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
584
585
586
</script>

{#if $showCallOverlay}
Timothy J. Baek's avatar
Timothy J. Baek committed
587
	<div class=" absolute w-full h-screen max-h-[100dvh] flex z-[999] overflow-hidden">
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
588
		<div
Timothy J. Baek's avatar
Timothy J. Baek committed
589
			class="absolute w-full h-screen max-h-[100dvh] bg-white text-gray-700 dark:bg-black dark:text-gray-300 flex justify-center"
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
590
		>
Timothy J. Baek's avatar
Timothy J. Baek committed
591
			<div class="max-w-lg w-full h-screen max-h-[100dvh] flex flex-col justify-between p-3 md:p-6">
Timothy J. Baek's avatar
Timothy J. Baek committed
592
				{#if camera}
593
594
595
596
597
598
599
600
601
					<button
						type="button"
						class="flex justify-center items-center w-full h-20 min-h-20"
						on:click={() => {
							if (assistantSpeaking) {
								stopAllAudio();
							}
						}}
					>
Timothy J. Baek's avatar
Timothy J. Baek committed
602
603
604
605
606
607
608
609
610
611
612
613
614
615
						{#if emoji}
							<div
								class="  transition-all rounded-full"
								style="font-size:{rmsLevel * 100 > 4
									? '4.5'
									: rmsLevel * 100 > 2
									? '4.25'
									: rmsLevel * 100 > 1
									? '3.75'
									: '3.5'}rem;width: 100%; text-align:center;"
							>
								{emoji}
							</div>
						{:else if loading}
Timothy J. Baek's avatar
Timothy J. Baek committed
616
617
618
619
620
621
622
623
							<svg
								class="size-12 text-gray-900 dark:text-gray-400"
								viewBox="0 0 24 24"
								fill="currentColor"
								xmlns="http://www.w3.org/2000/svg"
								><style>
									.spinner_qM83 {
										animation: spinner_8HQG 1.05s infinite;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
624
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
625
626
									.spinner_oXPr {
										animation-delay: 0.1s;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
627
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
628
629
									.spinner_ZTLf {
										animation-delay: 0.2s;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
630
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
									@keyframes spinner_8HQG {
										0%,
										57.14% {
											animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
											transform: translate(0);
										}
										28.57% {
											animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
											transform: translateY(-6px);
										}
										100% {
											transform: translate(0);
										}
									}
								</style><circle class="spinner_qM83" cx="4" cy="12" r="3" /><circle
									class="spinner_qM83 spinner_oXPr"
									cx="12"
									cy="12"
									r="3"
								/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
							>
						{:else}
							<div
								class=" {rmsLevel * 100 > 4
									? ' size-[4.5rem]'
									: rmsLevel * 100 > 2
									? ' size-16'
									: rmsLevel * 100 > 1
									? 'size-14'
									: 'size-12'}  transition-all bg-black dark:bg-white rounded-full"
							/>
						{/if}
						<!-- navbar -->
664
					</button>
Timothy J. Baek's avatar
Timothy J. Baek committed
665
666
				{/if}

Timothy J. Baek's avatar
Timothy J. Baek committed
667
				<div class="flex justify-center items-center flex-1 h-full w-full max-h-full">
Timothy J. Baek's avatar
Timothy J. Baek committed
668
					{#if !camera}
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
						<button
							type="button"
							on:click={() => {
								if (assistantSpeaking) {
									stopAllAudio();
								}
							}}
						>
							{#if emoji}
								<div
									class="  transition-all rounded-full"
									style="font-size:{rmsLevel * 100 > 4
										? '13'
										: rmsLevel * 100 > 2
										? '12'
										: rmsLevel * 100 > 1
										? '11.5'
										: '11'}rem;width:100%;text-align:center;"
								>
									{emoji}
								</div>
							{:else if loading}
								<svg
									class="size-44 text-gray-900 dark:text-gray-400"
									viewBox="0 0 24 24"
									fill="currentColor"
									xmlns="http://www.w3.org/2000/svg"
									><style>
										.spinner_qM83 {
											animation: spinner_8HQG 1.05s infinite;
Timothy J. Baek's avatar
Timothy J. Baek committed
699
										}
700
701
										.spinner_oXPr {
											animation-delay: 0.1s;
Timothy J. Baek's avatar
Timothy J. Baek committed
702
										}
703
704
										.spinner_ZTLf {
											animation-delay: 0.2s;
Timothy J. Baek's avatar
Timothy J. Baek committed
705
										}
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
										@keyframes spinner_8HQG {
											0%,
											57.14% {
												animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
												transform: translate(0);
											}
											28.57% {
												animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
												transform: translateY(-6px);
											}
											100% {
												transform: translate(0);
											}
										}
									</style><circle class="spinner_qM83" cx="4" cy="12" r="3" /><circle
										class="spinner_qM83 spinner_oXPr"
										cx="12"
										cy="12"
										r="3"
									/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
								>
							{:else}
								<div
									class=" {rmsLevel * 100 > 4
										? ' size-52'
										: rmsLevel * 100 > 2
										? 'size-48'
										: rmsLevel * 100 > 1
										? 'size-[11.5rem]'
										: 'size-44'}  transition-all bg-black dark:bg-white rounded-full"
								/>
							{/if}
						</button>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
739
					{:else}
Timothy J. Baek's avatar
Timothy J. Baek committed
740
741
742
						<div
							class="relative flex video-container w-full max-h-full pt-2 pb-4 md:py-6 px-2 h-full"
						>
Timothy J. Baek's avatar
Timothy J. Baek committed
743
744
745
							<video
								id="camera-feed"
								autoplay
Timothy J. Baek's avatar
Timothy J. Baek committed
746
								class="rounded-2xl h-full min-w-full object-cover object-center"
Timothy J. Baek's avatar
Timothy J. Baek committed
747
								playsinline
Timothy J. Baek's avatar
Timothy J. Baek committed
748
749
750
751
							/>

							<canvas id="camera-canvas" style="display:none;" />

Timothy J. Baek's avatar
Timothy J. Baek committed
752
							<div class=" absolute top-4 md:top-8 left-4">
Timothy J. Baek's avatar
Timothy J. Baek committed
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
								<button
									type="button"
									class="p-1.5 text-white cursor-pointer backdrop-blur-xl bg-black/10 rounded-full"
									on:click={() => {
										stopCamera();
									}}
								>
									<svg
										xmlns="http://www.w3.org/2000/svg"
										viewBox="0 0 16 16"
										fill="currentColor"
										class="size-6"
									>
										<path
											d="M5.28 4.22a.75.75 0 0 0-1.06 1.06L6.94 8l-2.72 2.72a.75.75 0 1 0 1.06 1.06L8 9.06l2.72 2.72a.75.75 0 1 0 1.06-1.06L9.06 8l2.72-2.72a.75.75 0 0 0-1.06-1.06L8 6.94 5.28 4.22Z"
										/>
									</svg>
								</button>
							</div>
						</div>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
773
					{/if}
Timothy J. Baek's avatar
Timothy J. Baek committed
774
775
				</div>

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
776
777
				<div class="flex justify-between items-center pb-2 w-full">
					<div>
Timothy J. Baek's avatar
Timothy J. Baek committed
778
779
780
781
782
783
784
785
						{#if camera}
							<VideoInputMenu
								devices={videoInputDevices}
								on:change={async (e) => {
									console.log(e.detail);
									selectedVideoInputDeviceId = e.detail;
									await stopVideoStream();
									await startVideoStream();
Timothy J. Baek's avatar
Timothy J. Baek committed
786
787
								}}
							>
Timothy J. Baek's avatar
Timothy J. Baek committed
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
								<button class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900" type="button">
									<svg
										xmlns="http://www.w3.org/2000/svg"
										viewBox="0 0 20 20"
										fill="currentColor"
										class="size-5"
									>
										<path
											fill-rule="evenodd"
											d="M15.312 11.424a5.5 5.5 0 0 1-9.201 2.466l-.312-.311h2.433a.75.75 0 0 0 0-1.5H3.989a.75.75 0 0 0-.75.75v4.242a.75.75 0 0 0 1.5 0v-2.43l.31.31a7 7 0 0 0 11.712-3.138.75.75 0 0 0-1.449-.39Zm1.23-3.723a.75.75 0 0 0 .219-.53V2.929a.75.75 0 0 0-1.5 0V5.36l-.31-.31A7 7 0 0 0 3.239 8.188a.75.75 0 1 0 1.448.389A5.5 5.5 0 0 1 13.89 6.11l.311.31h-2.432a.75.75 0 0 0 0 1.5h4.243a.75.75 0 0 0 .53-.219Z"
											clip-rule="evenodd"
										/>
									</svg>
								</button>
							</VideoInputMenu>
						{:else}
Timothy J. Baek's avatar
Timothy J. Baek committed
804
							<Tooltip content={$i18n.t('Camera')}>
Timothy J. Baek's avatar
Timothy J. Baek committed
805
806
807
								<button
									class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
									type="button"
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
808
809
									on:click={async () => {
										await navigator.mediaDevices.getUserMedia({ video: true });
Timothy J. Baek's avatar
Timothy J. Baek committed
810
811
										startCamera();
									}}
Timothy J. Baek's avatar
Timothy J. Baek committed
812
								>
Timothy J. Baek's avatar
Timothy J. Baek committed
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
									<svg
										xmlns="http://www.w3.org/2000/svg"
										fill="none"
										viewBox="0 0 24 24"
										stroke-width="1.5"
										stroke="currentColor"
										class="size-5"
									>
										<path
											stroke-linecap="round"
											stroke-linejoin="round"
											d="M6.827 6.175A2.31 2.31 0 0 1 5.186 7.23c-.38.054-.757.112-1.134.175C2.999 7.58 2.25 8.507 2.25 9.574V18a2.25 2.25 0 0 0 2.25 2.25h15A2.25 2.25 0 0 0 21.75 18V9.574c0-1.067-.75-1.994-1.802-2.169a47.865 47.865 0 0 0-1.134-.175 2.31 2.31 0 0 1-1.64-1.055l-.822-1.316a2.192 2.192 0 0 0-1.736-1.039 48.774 48.774 0 0 0-5.232 0 2.192 2.192 0 0 0-1.736 1.039l-.821 1.316Z"
										/>
										<path
											stroke-linecap="round"
											stroke-linejoin="round"
											d="M16.5 12.75a4.5 4.5 0 1 1-9 0 4.5 4.5 0 0 1 9 0ZM18.75 10.5h.008v.008h-.008V10.5Z"
										/>
									</svg>
								</button>
							</Tooltip>
						{/if}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
835
					</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
836

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
837
					<div>
838
839
840
841
842
843
844
845
						<button
							type="button"
							on:click={() => {
								if (assistantSpeaking) {
									stopAllAudio();
								}
							}}
						>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
846
847
							<div class=" line-clamp-1 text-sm font-medium">
								{#if loading}
Karl Lee's avatar
Karl Lee committed
848
									{$i18n.t('Thinking...')}
849
850
								{:else if assistantSpeaking}
									{$i18n.t('Tap to interrupt')}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
851
								{:else}
Karl Lee's avatar
Karl Lee committed
852
									{$i18n.t('Listening...')}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
853
854
855
856
857
858
859
860
861
862
863
864
								{/if}
							</div>
						</button>
					</div>

					<div>
						<button
							class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
							on:click={async () => {
								showCallOverlay.set(false);
							}}
							type="button"
Timothy J. Baek's avatar
Timothy J. Baek committed
865
						>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
866
867
868
869
870
871
872
873
874
875
876
877
							<svg
								xmlns="http://www.w3.org/2000/svg"
								viewBox="0 0 20 20"
								fill="currentColor"
								class="size-5"
							>
								<path
									d="M6.28 5.22a.75.75 0 0 0-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 1 0 1.06 1.06L10 11.06l3.72 3.72a.75.75 0 1 0 1.06-1.06L11.06 10l3.72-3.72a.75.75 0 0 0-1.06-1.06L10 8.94 6.28 5.22Z"
								/>
							</svg>
						</button>
					</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
878
879
880
881
				</div>
			</div>
		</div>
	</div>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
882
{/if}