CallOverlay.svelte 24.1 KB
Newer Older
Timothy J. Baek's avatar
Timothy J. Baek committed
1
<script lang="ts">
2
	import { config, models, settings, showCallOverlay } from '$lib/stores';
Timothy J. Baek's avatar
Timothy J. Baek committed
3
4
	import { onMount, tick, getContext } from 'svelte';

Timothy J. Baek's avatar
Timothy J. Baek committed
5
6
7
8
9
10
	import {
		blobToFile,
		calculateSHA256,
		extractSentencesForAudio,
		findWordIndices
	} from '$lib/utils';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
11
	import { generateEmoji } from '$lib/apis';
Timothy J. Baek's avatar
Timothy J. Baek committed
12
	import { synthesizeOpenAISpeech, transcribeAudio } from '$lib/apis/audio';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
13

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
14
	import { toast } from 'svelte-sonner';
Timothy J. Baek's avatar
Timothy J. Baek committed
15

Timothy J. Baek's avatar
Timothy J. Baek committed
16
	import Tooltip from '$lib/components/common/Tooltip.svelte';
Timothy J. Baek's avatar
Timothy J. Baek committed
17
	import VideoInputMenu from './CallOverlay/VideoInputMenu.svelte';
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
18

Timothy J. Baek's avatar
Timothy J. Baek committed
19
20
	const i18n = getContext('i18n');

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
21
	export let eventTarget: EventTarget;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
22

23
	export let submitPrompt: Function;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
24
25
	export let stopResponse: Function;

Timothy J. Baek's avatar
Timothy J. Baek committed
26
	export let files;
27

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
28
29
30
	export let chatId;
	export let modelId;

31
32
	let model = null;

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
33
34
	let loading = false;
	let confirmed = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
35
	let interrupted = false;
36
	let assistantSpeaking = false;
Timothy J. Baek's avatar
Timothy J. Baek committed
37

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
38
39
	let emoji = null;

Timothy J. Baek's avatar
Timothy J. Baek committed
40
41
42
	let camera = false;
	let cameraStream = null;

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
43
	let chatStreaming = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
44

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
45
46
	let rmsLevel = 0;
	let hasStartedSpeaking = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
47
48
49
	let mediaRecorder;
	let audioChunks = [];

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
	let videoInputDevices = [];
	let selectedVideoInputDeviceId = null;

	const getVideoInputDevices = async () => {
		const devices = await navigator.mediaDevices.enumerateDevices();
		videoInputDevices = devices.filter((device) => device.kind === 'videoinput');

		if (!!navigator.mediaDevices.getDisplayMedia) {
			videoInputDevices = [
				...videoInputDevices,
				{
					deviceId: 'screen',
					label: 'Screen Share'
				}
			];
		}

		console.log(videoInputDevices);
		if (selectedVideoInputDeviceId === null && videoInputDevices.length > 0) {
			selectedVideoInputDeviceId = videoInputDevices[0].deviceId;
		}
	};

	const startCamera = async () => {
		await getVideoInputDevices();

		if (cameraStream === null) {
			camera = true;
			await tick();
			try {
				await startVideoStream();
			} catch (err) {
				console.error('Error accessing webcam: ', err);
			}
		}
	};

	const startVideoStream = async () => {
		const video = document.getElementById('camera-feed');
		if (video) {
			if (selectedVideoInputDeviceId === 'screen') {
				cameraStream = await navigator.mediaDevices.getDisplayMedia({
					video: {
						cursor: 'always'
					},
					audio: false
				});
			} else {
				cameraStream = await navigator.mediaDevices.getUserMedia({
					video: {
						deviceId: selectedVideoInputDeviceId ? { exact: selectedVideoInputDeviceId } : undefined
					}
				});
			}

			if (cameraStream) {
				await getVideoInputDevices();
				video.srcObject = cameraStream;
				await video.play();
			}
		}
	};

	const stopVideoStream = async () => {
		if (cameraStream) {
			const tracks = cameraStream.getTracks();
			tracks.forEach((track) => track.stop());
		}

		cameraStream = null;
	};

	const takeScreenshot = () => {
		const video = document.getElementById('camera-feed');
		const canvas = document.getElementById('camera-canvas');

		if (!canvas) {
			return;
		}

		const context = canvas.getContext('2d');

		// Make the canvas match the video dimensions
		canvas.width = video.videoWidth;
		canvas.height = video.videoHeight;

		// Draw the image from the video onto the canvas
		context.drawImage(video, 0, 0, video.videoWidth, video.videoHeight);

		// Convert the canvas to a data base64 URL and console log it
		const dataURL = canvas.toDataURL('image/png');
		console.log(dataURL);

		return dataURL;
	};

	const stopCamera = async () => {
		await stopVideoStream();
		camera = false;
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
151
	const MIN_DECIBELS = -55;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
152
153
	const VISUALIZER_BUFFER_LENGTH = 300;

Timothy J. Baek's avatar
Timothy J. Baek committed
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
	const transcribeHandler = async (audioBlob) => {
		// Create a blob from the audio chunks

		await tick();
		const file = blobToFile(audioBlob, 'recording.wav');

		const res = await transcribeAudio(localStorage.token, file).catch((error) => {
			toast.error(error);
			return null;
		});

		if (res) {
			console.log(res.text);

			if (res.text !== '') {
				const _responses = await submitPrompt(res.text, { _raw: true });
				console.log(_responses);
			}
		}
	};

	const stopRecordingCallback = async (_continue = true) => {
		if ($showCallOverlay) {
			console.log('%c%s', 'color: red; font-size: 20px;', '🚨 stopRecordingCallback 🚨');

			// deep copy the audioChunks array
			const _audioChunks = audioChunks.slice(0);

			audioChunks = [];
			mediaRecorder = false;

			if (_continue) {
				startRecording();
			}

			if (confirmed) {
				loading = true;
				emoji = null;

				if (cameraStream) {
					const imageUrl = takeScreenshot();

					files = [
						{
							type: 'image',
							url: imageUrl
						}
					];
				}

				const audioBlob = new Blob(_audioChunks, { type: 'audio/wav' });
				await transcribeHandler(audioBlob);

				confirmed = false;
				loading = false;
			}
		} else {
			audioChunks = [];
			mediaRecorder = false;
		}
	};

	const startRecording = async () => {
		const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
		mediaRecorder = new MediaRecorder(stream);
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
219

Timothy J. Baek's avatar
Timothy J. Baek committed
220
221
222
223
224
		mediaRecorder.onstart = () => {
			console.log('Recording started');
			audioChunks = [];
			analyseAudio(stream);
		};
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
225

Timothy J. Baek's avatar
Timothy J. Baek committed
226
227
228
229
230
		mediaRecorder.ondataavailable = (event) => {
			if (hasStartedSpeaking) {
				audioChunks.push(event.data);
			}
		};
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
231
232
233
234

		mediaRecorder.onstop = (e) => {
			console.log('Recording stopped', e);
			stopRecordingCallback();
Timothy J. Baek's avatar
Timothy J. Baek committed
235
		};
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
236

Timothy J. Baek's avatar
Timothy J. Baek committed
237
238
239
		mediaRecorder.start();
	};

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
	// Function to calculate the RMS level from time domain data
	const calculateRMS = (data: Uint8Array) => {
		let sumSquares = 0;
		for (let i = 0; i < data.length; i++) {
			const normalizedValue = (data[i] - 128) / 128; // Normalize the data
			sumSquares += normalizedValue * normalizedValue;
		}
		return Math.sqrt(sumSquares / data.length);
	};

	const analyseAudio = (stream) => {
		const audioContext = new AudioContext();
		const audioStreamSource = audioContext.createMediaStreamSource(stream);

		const analyser = audioContext.createAnalyser();
		analyser.minDecibels = MIN_DECIBELS;
		audioStreamSource.connect(analyser);

		const bufferLength = analyser.frequencyBinCount;

		const domainData = new Uint8Array(bufferLength);
		const timeDomainData = new Uint8Array(analyser.fftSize);

		let lastSoundTime = Date.now();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
264
		hasStartedSpeaking = false;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
265

Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
266
267
		console.log('🔊 Sound detection started', lastSoundTime, hasStartedSpeaking);

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
268
269
		const detectSound = () => {
			const processFrame = () => {
Timothy J. Baek's avatar
Timothy J. Baek committed
270
271
272
				if (!mediaRecorder || !$showCallOverlay) {
					return;
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
273

274
				if (assistantSpeaking && !($settings?.voiceInterruption ?? false)) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
275
					// Mute the audio if the assistant is speaking
276
					analyser.maxDecibels = 0;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
277
					analyser.minDecibels = -1;
278
279
280
281
282
				} else {
					analyser.minDecibels = MIN_DECIBELS;
					analyser.maxDecibels = -30;
				}

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
283
284
285
286
287
288
289
290
291
				analyser.getByteTimeDomainData(timeDomainData);
				analyser.getByteFrequencyData(domainData);

				// Calculate RMS level from time domain data
				rmsLevel = calculateRMS(timeDomainData);

				// Check if initial speech/noise has started
				const hasSound = domainData.some((value) => value > 0);
				if (hasSound) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
292
293
					// BIG RED TEXT
					console.log('%c%s', 'color: red; font-size: 20px;', '🔊 Sound detected');
Timothy J. Baek's avatar
Timothy J. Baek committed
294
295
296
297
298
299
300

					if (!hasStartedSpeaking) {
						hasStartedSpeaking = true;
						stopAllAudio();
					}

					lastSoundTime = Date.now();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
301
302
303
304
305
306
307
308
				}

				// Start silence detection only after initial speech/noise has been detected
				if (hasStartedSpeaking) {
					if (Date.now() - lastSoundTime > 2000) {
						confirmed = true;

						if (mediaRecorder) {
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
309
							console.log('%c%s', 'color: red; font-size: 20px;', '🔇 Silence detected');
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
310
							mediaRecorder.stop();
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
311
							return;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
312
313
314
315
316
317
318
319
320
						}
					}
				}

				window.requestAnimationFrame(processFrame);
			};

			window.requestAnimationFrame(processFrame);
		};
Timothy J. Baek's avatar
Timothy J. Baek committed
321

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
322
323
324
		detectSound();
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
325
326
327
	let finishedMessages = {};
	let currentMessageId = null;
	let currentUtterance = null;
Timothy J. Baek's avatar
Timothy J. Baek committed
328

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
	const speakSpeechSynthesisHandler = (content) => {
		if ($showCallOverlay) {
			return new Promise((resolve) => {
				let voices = [];
				const getVoicesLoop = setInterval(async () => {
					voices = await speechSynthesis.getVoices();
					if (voices.length > 0) {
						clearInterval(getVoicesLoop);

						const voice =
							voices
								?.filter(
									(v) => v.voiceURI === ($settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice)
								)
								?.at(0) ?? undefined;

						currentUtterance = new SpeechSynthesisUtterance(content);

						if (voice) {
							currentUtterance.voice = voice;
						}

						speechSynthesis.speak(currentUtterance);
						currentUtterance.onend = async (e) => {
Timothy J. Baek's avatar
Timothy J. Baek committed
353
							await new Promise((r) => setTimeout(r, 200));
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
354
355
356
357
358
359
360
361
362
363
364
							resolve(e);
						};
					}
				}, 100);
			});
		} else {
			return Promise.resolve();
		}
	};

	const playAudio = (audio) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
365
		if ($showCallOverlay) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
366
			return new Promise((resolve) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
367
				const audioElement = document.getElementById('audioElement');
368

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
369
				if (audioElement) {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
370
					audioElement.src = audio.src;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
371
					audioElement.muted = true;
Timothy J. Baek's avatar
Timothy J. Baek committed
372

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
373
374
375
376
377
378
					audioElement
						.play()
						.then(() => {
							audioElement.muted = false;
						})
						.catch((error) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
379
							console.error(error);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
380
						});
381

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
382
					audioElement.onended = async (e) => {
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
383
384
						await new Promise((r) => setTimeout(r, 100));
						resolve(e);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
385
386
					};
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
387
388
389
390
			});
		} else {
			return Promise.resolve();
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
391
392
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
393
	const stopAllAudio = async () => {
394
		assistantSpeaking = false;
Timothy J. Baek's avatar
Timothy J. Baek committed
395
		interrupted = true;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
396

Timothy J. Baek's avatar
Timothy J. Baek committed
397
398
		if (chatStreaming) {
			stopResponse();
Timothy J. Baek's avatar
Timothy J. Baek committed
399
400
		}

Timothy J. Baek's avatar
Timothy J. Baek committed
401
402
403
404
		if (currentUtterance) {
			speechSynthesis.cancel();
			currentUtterance = null;
		}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
405

Timothy J. Baek's avatar
Timothy J. Baek committed
406
407
		const audioElement = document.getElementById('audioElement');
		if (audioElement) {
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
408
			audioElement.muted = true;
Timothy J. Baek's avatar
Timothy J. Baek committed
409
410
411
412
			audioElement.pause();
			audioElement.currentTime = 0;
		}
	};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
413

Timothy J. Baek's avatar
Timothy J. Baek committed
414
	let audioAbortController = new AbortController();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
415

Timothy J. Baek's avatar
Timothy J. Baek committed
416
417
	// Audio cache map where key is the content and value is the Audio object.
	const audioCache = new Map();
Timothy J. Baek's avatar
Timothy J. Baek committed
418
419
	const emojiCache = new Map();

Timothy J. Baek's avatar
Timothy J. Baek committed
420
421
422
	const fetchAudio = async (content) => {
		if (!audioCache.has(content)) {
			try {
Timothy J. Baek's avatar
Timothy J. Baek committed
423
424
425
426
427
428
429
				// Set the emoji for the content if needed
				if ($settings?.showEmojiInCall ?? false) {
					const emoji = await generateEmoji(localStorage.token, modelId, content, chatId);
					if (emoji) {
						emojiCache.set(content, emoji);
					}
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
430

Timothy J. Baek's avatar
Timothy J. Baek committed
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
				if ($config.audio.tts.engine !== '') {
					const res = await synthesizeOpenAISpeech(
						localStorage.token,
						$settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice,
						content
					).catch((error) => {
						console.error(error);
						return null;
					});

					if (res) {
						const blob = await res.blob();
						const blobUrl = URL.createObjectURL(blob);
						audioCache.set(content, new Audio(blobUrl));
					}
				} else {
					audioCache.set(content, true);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
448
				}
Timothy J. Baek's avatar
Timothy J. Baek committed
449
450
			} catch (error) {
				console.error('Error synthesizing speech:', error);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
451
452
			}
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
453

Timothy J. Baek's avatar
Timothy J. Baek committed
454
		return audioCache.get(content);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
455
	};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
456

Timothy J. Baek's avatar
Timothy J. Baek committed
457
458
459
460
461
462
463
464
465
466
	let messages = {};

	const monitorAndPlayAudio = async (id, signal) => {
		while (!signal.aborted) {
			if (messages[id] && messages[id].length > 0) {
				// Retrieve the next content string from the queue
				const content = messages[id].shift(); // Dequeues the content for playing

				if (audioCache.has(content)) {
					// If content is available in the cache, play it
Timothy J. Baek's avatar
Timothy J. Baek committed
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491

					// Set the emoji for the content if available
					if (($settings?.showEmojiInCall ?? false) && emojiCache.has(content)) {
						emoji = emojiCache.get(content);
					} else {
						emoji = null;
					}

					if ($config.audio.tts.engine !== '') {
						try {
							console.log(
								'%c%s',
								'color: red; font-size: 20px;',
								`Playing audio for content: ${content}`
							);

							const audio = audioCache.get(content);
							await playAudio(audio); // Here ensure that playAudio is indeed correct method to execute
							console.log(`Played audio for content: ${content}`);
							await new Promise((resolve) => setTimeout(resolve, 200)); // Wait before retrying to reduce tight loop
						} catch (error) {
							console.error('Error playing audio:', error);
						}
					} else {
						await speakSpeechSynthesisHandler(content);
Timothy J. Baek's avatar
Timothy J. Baek committed
492
493
494
495
496
497
					}
				} else {
					// If not available in the cache, push it back to the queue and delay
					messages[id].unshift(content); // Re-queue the content at the start
					console.log(`Audio for "${content}" not yet available in the cache, re-queued...`);
					await new Promise((resolve) => setTimeout(resolve, 200)); // Wait before retrying to reduce tight loop
Timothy J. Baek's avatar
Timothy J. Baek committed
498
				}
Timothy J. Baek's avatar
Timothy J. Baek committed
499
500
			} else if (finishedMessages[id] && messages[id] && messages[id].length === 0) {
				// If the message is finished and there are no more messages to process, break the loop
501
				assistantSpeaking = false;
Timothy J. Baek's avatar
Timothy J. Baek committed
502
503
504
505
				break;
			} else {
				// No messages to process, sleep for a bit
				await new Promise((resolve) => setTimeout(resolve, 200));
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
506
507
			}
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
508
		console.log(`Audio monitoring and playing stopped for message ID ${id}`);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
509
510
	};

Timothy J. Baek's avatar
Timothy J. Baek committed
511
	onMount(async () => {
512
513
		model = $models.find((m) => m.id === modelId);

Timothy J. Baek's avatar
Timothy J. Baek committed
514
		startRecording();
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
515

Timothy J. Baek's avatar
Timothy J. Baek committed
516
517
		const chatStartHandler = async (e) => {
			const { id } = e.detail;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
518

Timothy J. Baek's avatar
Timothy J. Baek committed
519
			chatStreaming = true;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
520

Timothy J. Baek's avatar
Timothy J. Baek committed
521
522
			if (currentMessageId !== id) {
				console.log(`Received chat start event for message ID ${id}`);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
523

Timothy J. Baek's avatar
Timothy J. Baek committed
524
525
526
				currentMessageId = id;
				if (audioAbortController) {
					audioAbortController.abort();
Timothy J. Baek's avatar
Timothy J. Baek committed
527
				}
Timothy J. Baek's avatar
Timothy J. Baek committed
528
529
				audioAbortController = new AbortController();

530
				assistantSpeaking = true;
Timothy J. Baek's avatar
Timothy J. Baek committed
531
532
				// Start monitoring and playing audio for the message ID
				monitorAndPlayAudio(id, audioAbortController.signal);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
533
			}
Timothy J. Baek's avatar
Timothy J. Baek committed
534
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
535

Timothy J. Baek's avatar
Timothy J. Baek committed
536
537
538
539
540
541
542
		const chatEventHandler = async (e) => {
			const { id, content } = e.detail;
			// "id" here is message id
			// if "id" is not the same as "currentMessageId" then do not process
			// "content" here is a sentence from the assistant,
			// there will be many sentences for the same "id"

Timothy J. Baek's avatar
Timothy J. Baek committed
543
544
			if (currentMessageId === id) {
				console.log(`Received chat event for message ID ${id}: ${content}`);
Timothy J. Baek's avatar
Timothy J. Baek committed
545

Timothy J. Baek's avatar
Timothy J. Baek committed
546
547
548
549
550
551
				try {
					if (messages[id] === undefined) {
						messages[id] = [content];
					} else {
						messages[id].push(content);
					}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
552

Timothy J. Baek's avatar
Timothy J. Baek committed
553
					console.log(content);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
554

Timothy J. Baek's avatar
Timothy J. Baek committed
555
556
557
					fetchAudio(content);
				} catch (error) {
					console.error('Failed to fetch or play audio:', error);
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
558
				}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
559
			}
Timothy J. Baek's avatar
Timothy J. Baek committed
560
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
561

Timothy J. Baek's avatar
Timothy J. Baek committed
562
563
564
		const chatFinishHandler = async (e) => {
			const { id, content } = e.detail;
			// "content" here is the entire message from the assistant
565
			finishedMessages[id] = true;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
566

Timothy J. Baek's avatar
Timothy J. Baek committed
567
568
569
570
571
572
573
574
575
576
577
578
			chatStreaming = false;
		};

		eventTarget.addEventListener('chat:start', chatStartHandler);
		eventTarget.addEventListener('chat', chatEventHandler);
		eventTarget.addEventListener('chat:finish', chatFinishHandler);

		return async () => {
			eventTarget.removeEventListener('chat:start', chatStartHandler);
			eventTarget.removeEventListener('chat', chatEventHandler);
			eventTarget.removeEventListener('chat:finish', chatFinishHandler);

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
579
580
			audioAbortController.abort();
			await tick();
Timothy J. Baek's avatar
fix  
Timothy J. Baek committed
581

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
582
583
			await stopAllAudio();

Timothy J. Baek's avatar
Timothy J. Baek committed
584
585
586
			await stopRecordingCallback(false);
			await stopCamera();
		};
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
587
	});
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
588
589
590
</script>

{#if $showCallOverlay}
Timothy J. Baek's avatar
Timothy J. Baek committed
591
	<div class=" absolute w-full h-screen max-h-[100dvh] flex z-[999] overflow-hidden">
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
592
		<div
Timothy J. Baek's avatar
Timothy J. Baek committed
593
			class="absolute w-full h-screen max-h-[100dvh] bg-white text-gray-700 dark:bg-black dark:text-gray-300 flex justify-center"
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
594
		>
Timothy J. Baek's avatar
Timothy J. Baek committed
595
			<div class="max-w-lg w-full h-screen max-h-[100dvh] flex flex-col justify-between p-3 md:p-6">
Timothy J. Baek's avatar
Timothy J. Baek committed
596
				{#if camera}
597
598
599
600
601
602
603
604
605
					<button
						type="button"
						class="flex justify-center items-center w-full h-20 min-h-20"
						on:click={() => {
							if (assistantSpeaking) {
								stopAllAudio();
							}
						}}
					>
Timothy J. Baek's avatar
Timothy J. Baek committed
606
607
608
609
610
611
612
613
614
615
616
617
618
						{#if emoji}
							<div
								class="  transition-all rounded-full"
								style="font-size:{rmsLevel * 100 > 4
									? '4.5'
									: rmsLevel * 100 > 2
									? '4.25'
									: rmsLevel * 100 > 1
									? '3.75'
									: '3.5'}rem;width: 100%; text-align:center;"
							>
								{emoji}
							</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
619
						{:else if loading || assistantSpeaking}
Timothy J. Baek's avatar
Timothy J. Baek committed
620
621
622
623
624
625
626
627
							<svg
								class="size-12 text-gray-900 dark:text-gray-400"
								viewBox="0 0 24 24"
								fill="currentColor"
								xmlns="http://www.w3.org/2000/svg"
								><style>
									.spinner_qM83 {
										animation: spinner_8HQG 1.05s infinite;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
628
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
629
630
									.spinner_oXPr {
										animation-delay: 0.1s;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
631
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
632
633
									.spinner_ZTLf {
										animation-delay: 0.2s;
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
634
									}
Timothy J. Baek's avatar
Timothy J. Baek committed
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
									@keyframes spinner_8HQG {
										0%,
										57.14% {
											animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
											transform: translate(0);
										}
										28.57% {
											animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
											transform: translateY(-6px);
										}
										100% {
											transform: translate(0);
										}
									}
								</style><circle class="spinner_qM83" cx="4" cy="12" r="3" /><circle
									class="spinner_qM83 spinner_oXPr"
									cx="12"
									cy="12"
									r="3"
								/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
							>
						{:else}
							<div
								class=" {rmsLevel * 100 > 4
									? ' size-[4.5rem]'
									: rmsLevel * 100 > 2
									? ' size-16'
									: rmsLevel * 100 > 1
									? 'size-14'
664
665
666
667
668
669
670
									: 'size-12'}  transition-all rounded-full {(model?.info?.meta
									?.profile_image_url ?? '/favicon.png') !== '/favicon.png'
									? ' bg-cover bg-center bg-no-repeat'
									: 'bg-black dark:bg-white'}  bg-black dark:bg-white"
								style={(model?.info?.meta?.profile_image_url ?? '/favicon.png') !== '/favicon.png'
									? `background-image: url('${model?.info?.meta?.profile_image_url}');`
									: ''}
Timothy J. Baek's avatar
Timothy J. Baek committed
671
672
673
							/>
						{/if}
						<!-- navbar -->
674
					</button>
Timothy J. Baek's avatar
Timothy J. Baek committed
675
676
				{/if}

Timothy J. Baek's avatar
Timothy J. Baek committed
677
				<div class="flex justify-center items-center flex-1 h-full w-full max-h-full">
Timothy J. Baek's avatar
Timothy J. Baek committed
678
					{#if !camera}
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
						<button
							type="button"
							on:click={() => {
								if (assistantSpeaking) {
									stopAllAudio();
								}
							}}
						>
							{#if emoji}
								<div
									class="  transition-all rounded-full"
									style="font-size:{rmsLevel * 100 > 4
										? '13'
										: rmsLevel * 100 > 2
										? '12'
										: rmsLevel * 100 > 1
										? '11.5'
										: '11'}rem;width:100%;text-align:center;"
								>
									{emoji}
								</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
700
							{:else if loading || assistantSpeaking}
701
702
703
704
705
706
707
708
								<svg
									class="size-44 text-gray-900 dark:text-gray-400"
									viewBox="0 0 24 24"
									fill="currentColor"
									xmlns="http://www.w3.org/2000/svg"
									><style>
										.spinner_qM83 {
											animation: spinner_8HQG 1.05s infinite;
Timothy J. Baek's avatar
Timothy J. Baek committed
709
										}
710
711
										.spinner_oXPr {
											animation-delay: 0.1s;
Timothy J. Baek's avatar
Timothy J. Baek committed
712
										}
713
714
										.spinner_ZTLf {
											animation-delay: 0.2s;
Timothy J. Baek's avatar
Timothy J. Baek committed
715
										}
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
										@keyframes spinner_8HQG {
											0%,
											57.14% {
												animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
												transform: translate(0);
											}
											28.57% {
												animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
												transform: translateY(-6px);
											}
											100% {
												transform: translate(0);
											}
										}
									</style><circle class="spinner_qM83" cx="4" cy="12" r="3" /><circle
										class="spinner_qM83 spinner_oXPr"
										cx="12"
										cy="12"
										r="3"
									/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
								>
							{:else}
								<div
									class=" {rmsLevel * 100 > 4
										? ' size-52'
										: rmsLevel * 100 > 2
										? 'size-48'
										: rmsLevel * 100 > 1
										? 'size-[11.5rem]'
745
746
747
748
749
750
751
										: 'size-44'}  transition-all rounded-full {(model?.info?.meta
										?.profile_image_url ?? '/favicon.png') !== '/favicon.png'
										? ' bg-cover bg-center bg-no-repeat'
										: 'bg-black dark:bg-white'} "
									style={(model?.info?.meta?.profile_image_url ?? '/favicon.png') !== '/favicon.png'
										? `background-image: url('${model?.info?.meta?.profile_image_url}');`
										: ''}
752
753
754
								/>
							{/if}
						</button>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
755
					{:else}
Timothy J. Baek's avatar
Timothy J. Baek committed
756
757
758
						<div
							class="relative flex video-container w-full max-h-full pt-2 pb-4 md:py-6 px-2 h-full"
						>
Timothy J. Baek's avatar
Timothy J. Baek committed
759
760
761
							<video
								id="camera-feed"
								autoplay
Timothy J. Baek's avatar
Timothy J. Baek committed
762
								class="rounded-2xl h-full min-w-full object-cover object-center"
Timothy J. Baek's avatar
Timothy J. Baek committed
763
								playsinline
Timothy J. Baek's avatar
Timothy J. Baek committed
764
765
766
767
							/>

							<canvas id="camera-canvas" style="display:none;" />

Timothy J. Baek's avatar
Timothy J. Baek committed
768
							<div class=" absolute top-4 md:top-8 left-4">
Timothy J. Baek's avatar
Timothy J. Baek committed
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
								<button
									type="button"
									class="p-1.5 text-white cursor-pointer backdrop-blur-xl bg-black/10 rounded-full"
									on:click={() => {
										stopCamera();
									}}
								>
									<svg
										xmlns="http://www.w3.org/2000/svg"
										viewBox="0 0 16 16"
										fill="currentColor"
										class="size-6"
									>
										<path
											d="M5.28 4.22a.75.75 0 0 0-1.06 1.06L6.94 8l-2.72 2.72a.75.75 0 1 0 1.06 1.06L8 9.06l2.72 2.72a.75.75 0 1 0 1.06-1.06L9.06 8l2.72-2.72a.75.75 0 0 0-1.06-1.06L8 6.94 5.28 4.22Z"
										/>
									</svg>
								</button>
							</div>
						</div>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
789
					{/if}
Timothy J. Baek's avatar
Timothy J. Baek committed
790
791
				</div>

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
792
793
				<div class="flex justify-between items-center pb-2 w-full">
					<div>
Timothy J. Baek's avatar
Timothy J. Baek committed
794
795
796
797
798
799
800
801
						{#if camera}
							<VideoInputMenu
								devices={videoInputDevices}
								on:change={async (e) => {
									console.log(e.detail);
									selectedVideoInputDeviceId = e.detail;
									await stopVideoStream();
									await startVideoStream();
Timothy J. Baek's avatar
Timothy J. Baek committed
802
803
								}}
							>
Timothy J. Baek's avatar
Timothy J. Baek committed
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
								<button class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900" type="button">
									<svg
										xmlns="http://www.w3.org/2000/svg"
										viewBox="0 0 20 20"
										fill="currentColor"
										class="size-5"
									>
										<path
											fill-rule="evenodd"
											d="M15.312 11.424a5.5 5.5 0 0 1-9.201 2.466l-.312-.311h2.433a.75.75 0 0 0 0-1.5H3.989a.75.75 0 0 0-.75.75v4.242a.75.75 0 0 0 1.5 0v-2.43l.31.31a7 7 0 0 0 11.712-3.138.75.75 0 0 0-1.449-.39Zm1.23-3.723a.75.75 0 0 0 .219-.53V2.929a.75.75 0 0 0-1.5 0V5.36l-.31-.31A7 7 0 0 0 3.239 8.188a.75.75 0 1 0 1.448.389A5.5 5.5 0 0 1 13.89 6.11l.311.31h-2.432a.75.75 0 0 0 0 1.5h4.243a.75.75 0 0 0 .53-.219Z"
											clip-rule="evenodd"
										/>
									</svg>
								</button>
							</VideoInputMenu>
						{:else}
Timothy J. Baek's avatar
Timothy J. Baek committed
820
							<Tooltip content={$i18n.t('Camera')}>
Timothy J. Baek's avatar
Timothy J. Baek committed
821
822
823
								<button
									class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
									type="button"
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
824
825
									on:click={async () => {
										await navigator.mediaDevices.getUserMedia({ video: true });
Timothy J. Baek's avatar
Timothy J. Baek committed
826
827
										startCamera();
									}}
Timothy J. Baek's avatar
Timothy J. Baek committed
828
								>
Timothy J. Baek's avatar
Timothy J. Baek committed
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
									<svg
										xmlns="http://www.w3.org/2000/svg"
										fill="none"
										viewBox="0 0 24 24"
										stroke-width="1.5"
										stroke="currentColor"
										class="size-5"
									>
										<path
											stroke-linecap="round"
											stroke-linejoin="round"
											d="M6.827 6.175A2.31 2.31 0 0 1 5.186 7.23c-.38.054-.757.112-1.134.175C2.999 7.58 2.25 8.507 2.25 9.574V18a2.25 2.25 0 0 0 2.25 2.25h15A2.25 2.25 0 0 0 21.75 18V9.574c0-1.067-.75-1.994-1.802-2.169a47.865 47.865 0 0 0-1.134-.175 2.31 2.31 0 0 1-1.64-1.055l-.822-1.316a2.192 2.192 0 0 0-1.736-1.039 48.774 48.774 0 0 0-5.232 0 2.192 2.192 0 0 0-1.736 1.039l-.821 1.316Z"
										/>
										<path
											stroke-linecap="round"
											stroke-linejoin="round"
											d="M16.5 12.75a4.5 4.5 0 1 1-9 0 4.5 4.5 0 0 1 9 0ZM18.75 10.5h.008v.008h-.008V10.5Z"
										/>
									</svg>
								</button>
							</Tooltip>
						{/if}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
851
					</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
852

Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
853
					<div>
854
855
856
857
858
859
860
861
						<button
							type="button"
							on:click={() => {
								if (assistantSpeaking) {
									stopAllAudio();
								}
							}}
						>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
862
863
							<div class=" line-clamp-1 text-sm font-medium">
								{#if loading}
Karl Lee's avatar
Karl Lee committed
864
									{$i18n.t('Thinking...')}
865
866
								{:else if assistantSpeaking}
									{$i18n.t('Tap to interrupt')}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
867
								{:else}
Karl Lee's avatar
Karl Lee committed
868
									{$i18n.t('Listening...')}
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
869
870
871
872
873
874
875
876
877
878
879
880
								{/if}
							</div>
						</button>
					</div>

					<div>
						<button
							class=" p-3 rounded-full bg-gray-50 dark:bg-gray-900"
							on:click={async () => {
								showCallOverlay.set(false);
							}}
							type="button"
Timothy J. Baek's avatar
Timothy J. Baek committed
881
						>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
882
883
884
885
886
887
888
889
890
891
892
893
							<svg
								xmlns="http://www.w3.org/2000/svg"
								viewBox="0 0 20 20"
								fill="currentColor"
								class="size-5"
							>
								<path
									d="M6.28 5.22a.75.75 0 0 0-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 1 0 1.06 1.06L10 11.06l3.72 3.72a.75.75 0 1 0 1.06-1.06L11.06 10l3.72-3.72a.75.75 0 0 0-1.06-1.06L10 8.94 6.28 5.22Z"
								/>
							</svg>
						</button>
					</div>
Timothy J. Baek's avatar
Timothy J. Baek committed
894
895
896
897
				</div>
			</div>
		</div>
	</div>
Timothy J. Baek's avatar
refac  
Timothy J. Baek committed
898
{/if}