index.ts 11.5 KB
Newer Older
1
2
import { v4 as uuidv4 } from 'uuid';
import sha256 from 'js-sha256';
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import { getOllamaModels } from '$lib/apis/ollama';
import { getOpenAIModels } from '$lib/apis/openai';
import { getLiteLLMModels } from '$lib/apis/litellm';

export const getModels = async (token: string) => {
	let models = await Promise.all([
		await getOllamaModels(token).catch((error) => {
			console.log(error);
			return null;
		}),
		await getOpenAIModels(token).catch((error) => {
			console.log(error);
			return null;
		}),
		await getLiteLLMModels(token).catch((error) => {
			console.log(error);
			return null;
		})
	]);

	models = models
		.filter((models) => models)
		.reduce((a, e, i, arr) => a.concat(e, ...(i < arr.length - 1 ? [{ name: 'hr' }] : [])), []);

	return models;
};
29
30
31
32
33

//////////////////////////
// Helper functions
//////////////////////////

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
export const sanitizeResponseContent = (content: string) => {
	return content
		.replace(/<\|[a-z]*$/, '')
		.replace(/<\|[a-z]+\|$/, '')
		.replace(/<$/, '')
		.replaceAll(/<\|[a-z]+\|>/g, ' ')
		.replaceAll(/<br\s?\/?>/gi, '\n')
		.replaceAll('<', '&lt;')
		.trim();
};

export const revertSanitizedResponseContent = (content: string) => {
	return content.replaceAll('&lt;', '<');
};

Timothy J. Baek's avatar
Timothy J. Baek committed
49
50
51
52
export const capitalizeFirstLetter = (string) => {
	return string.charAt(0).toUpperCase() + string.slice(1);
};

53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
export const splitStream = (splitOn) => {
	let buffer = '';
	return new TransformStream({
		transform(chunk, controller) {
			buffer += chunk;
			const parts = buffer.split(splitOn);
			parts.slice(0, -1).forEach((part) => controller.enqueue(part));
			buffer = parts[parts.length - 1];
		},
		flush(controller) {
			if (buffer) controller.enqueue(buffer);
		}
	});
};

export const convertMessagesToHistory = (messages) => {
69
	const history = {
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
		messages: {},
		currentId: null
	};

	let parentMessageId = null;
	let messageId = null;

	for (const message of messages) {
		messageId = uuidv4();

		if (parentMessageId !== null) {
			history.messages[parentMessageId].childrenIds = [
				...history.messages[parentMessageId].childrenIds,
				messageId
			];
		}

		history.messages[messageId] = {
			...message,
			id: messageId,
			parentId: parentMessageId,
			childrenIds: []
		};

		parentMessageId = messageId;
	}

	history.currentId = messageId;
	return history;
};

export const getGravatarURL = (email) => {
	// Trim leading and trailing whitespace from
	// an email address and force all characters
	// to lower case
	const address = String(email).trim().toLowerCase();

	// Create a SHA256 hash of the final string
	const hash = sha256(address);

	// Grab the actual image URL
	return `https://www.gravatar.com/avatar/${hash}`;
};
Timothy J. Baek's avatar
Timothy J. Baek committed
113

114
115
116
export const canvasPixelTest = () => {
	// Test a 1x1 pixel to potentially identify browser/plugin fingerprint blocking or spoofing
	// Inspiration: https://github.com/kkapsner/CanvasBlocker/blob/master/test/detectionTest.js
Danny Liu's avatar
Danny Liu committed
117
	const canvas = document.createElement('canvas');
118
119
120
121
122
123
124
	const ctx = canvas.getContext('2d');
	canvas.height = 1;
	canvas.width = 1;
	const imageData = new ImageData(canvas.width, canvas.height);
	const pixelValues = imageData.data;

	// Generate RGB test data
Danny Liu's avatar
Danny Liu committed
125
126
	for (let i = 0; i < imageData.data.length; i += 1) {
		if (i % 4 !== 3) {
127
			pixelValues[i] = Math.floor(256 * Math.random());
Danny Liu's avatar
Danny Liu committed
128
		} else {
129
130
131
132
133
134
135
136
			pixelValues[i] = 255;
		}
	}

	ctx.putImageData(imageData, 0, 0);
	const p = ctx.getImageData(0, 0, canvas.width, canvas.height).data;

	// Read RGB data and fail if unmatched
Danny Liu's avatar
Danny Liu committed
137
138
139
140
141
142
143
144
145
146
147
	for (let i = 0; i < p.length; i += 1) {
		if (p[i] !== pixelValues[i]) {
			console.log(
				'canvasPixelTest: Wrong canvas pixel RGB value detected:',
				p[i],
				'at:',
				i,
				'expected:',
				pixelValues[i]
			);
			console.log('canvasPixelTest: Canvas blocking or spoofing is likely');
148
149
150
151
152
			return false;
		}
	}

	return true;
Danny Liu's avatar
Danny Liu committed
153
};
154

155
export const generateInitialsImage = (name) => {
Danny Liu's avatar
Danny Liu committed
156
157
158
159
	const canvas = document.createElement('canvas');
	const ctx = canvas.getContext('2d');
	canvas.width = 100;
	canvas.height = 100;
160

161
	if (!canvasPixelTest()) {
Danny Liu's avatar
Danny Liu committed
162
163
164
		console.log(
			'generateInitialsImage: failed pixel test, fingerprint evasion is likely. Using default image.'
		);
165
166
167
		return '/user.png';
	}

Danny Liu's avatar
Danny Liu committed
168
169
	ctx.fillStyle = '#F39C12';
	ctx.fillRect(0, 0, canvas.width, canvas.height);
170

Danny Liu's avatar
Danny Liu committed
171
172
173
174
	ctx.fillStyle = '#FFFFFF';
	ctx.font = '40px Helvetica';
	ctx.textAlign = 'center';
	ctx.textBaseline = 'middle';
175

Danny Liu's avatar
Danny Liu committed
176
	const sanitizedName = name.trim();
Danny Liu's avatar
Danny Liu committed
177
178
179
180
181
182
183
	const initials =
		sanitizedName.length > 0
			? sanitizedName[0] +
			  (sanitizedName.split(' ').length > 1
					? sanitizedName[sanitizedName.lastIndexOf(' ') + 1]
					: '')
			: '';
Danny Liu's avatar
Danny Liu committed
184

Danny Liu's avatar
Danny Liu committed
185
	ctx.fillText(initials.toUpperCase(), canvas.width / 2, canvas.height / 2);
186

Danny Liu's avatar
Danny Liu committed
187
	return canvas.toDataURL();
188
189
};

Timothy J. Baek's avatar
Timothy J. Baek committed
190
export const copyToClipboard = (text) => {
Timothy J. Baek's avatar
Timothy J. Baek committed
191
	if (!navigator.clipboard) {
Timothy J. Baek's avatar
Timothy J. Baek committed
192
		const textArea = document.createElement('textarea');
Timothy J. Baek's avatar
Timothy J. Baek committed
193
194
195
196
197
198
199
200
201
202
203
204
		textArea.value = text;

		// Avoid scrolling to bottom
		textArea.style.top = '0';
		textArea.style.left = '0';
		textArea.style.position = 'fixed';

		document.body.appendChild(textArea);
		textArea.focus();
		textArea.select();

		try {
Timothy J. Baek's avatar
Timothy J. Baek committed
205
206
			const successful = document.execCommand('copy');
			const msg = successful ? 'successful' : 'unsuccessful';
Timothy J. Baek's avatar
Timothy J. Baek committed
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
			console.log('Fallback: Copying text command was ' + msg);
		} catch (err) {
			console.error('Fallback: Oops, unable to copy', err);
		}

		document.body.removeChild(textArea);
		return;
	}
	navigator.clipboard.writeText(text).then(
		function () {
			console.log('Async: Copying to clipboard was successful!');
		},
		function (err) {
			console.error('Async: Could not copy text: ', err);
		}
	);
};
Timothy J. Baek's avatar
Timothy J. Baek committed
224

Timothy J. Baek's avatar
Timothy J. Baek committed
225
export const compareVersion = (latest, current) => {
226
	return current === '0.0.0'
Timothy J. Baek's avatar
Timothy J. Baek committed
227
		? false
Timothy J. Baek's avatar
Timothy J. Baek committed
228
		: current.localeCompare(latest, undefined, {
229
230
231
232
				numeric: true,
				sensitivity: 'case',
				caseFirst: 'upper'
		  }) < 0;
Timothy J. Baek's avatar
Timothy J. Baek committed
233
};
Timothy J. Baek's avatar
Timothy J. Baek committed
234
235
236

export const findWordIndices = (text) => {
	const regex = /\[([^\]]+)\]/g;
237
	const matches = [];
Timothy J. Baek's avatar
Timothy J. Baek committed
238
239
240
241
242
243
244
245
246
247
248
249
	let match;

	while ((match = regex.exec(text)) !== null) {
		matches.push({
			word: match[1],
			startIndex: match.index,
			endIndex: regex.lastIndex - 1
		});
	}

	return matches;
};
250

Timothy J. Baek's avatar
Timothy J. Baek committed
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
export const removeFirstHashWord = (inputString) => {
	// Split the string into an array of words
	const words = inputString.split(' ');

	// Find the index of the first word that starts with #
	const index = words.findIndex((word) => word.startsWith('#'));

	// Remove the first word with #
	if (index !== -1) {
		words.splice(index, 1);
	}

	// Join the remaining words back into a string
	const resultString = words.join(' ');

	return resultString;
};

269
270
271
272
273
274
275
276
277
278
279
280
281
export const transformFileName = (fileName) => {
	// Convert to lowercase
	const lowerCaseFileName = fileName.toLowerCase();

	// Remove special characters using regular expression
	const sanitizedFileName = lowerCaseFileName.replace(/[^\w\s]/g, '');

	// Replace spaces with dashes
	const finalFileName = sanitizedFileName.replace(/\s+/g, '-');

	return finalFileName;
};

282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
export const calculateSHA256 = async (file) => {
	// Create a FileReader to read the file asynchronously
	const reader = new FileReader();

	// Define a promise to handle the file reading
	const readFile = new Promise((resolve, reject) => {
		reader.onload = () => resolve(reader.result);
		reader.onerror = reject;
	});

	// Read the file as an ArrayBuffer
	reader.readAsArrayBuffer(file);

	try {
		// Wait for the FileReader to finish reading the file
		const buffer = await readFile;

		// Convert the ArrayBuffer to a Uint8Array
		const uint8Array = new Uint8Array(buffer);

		// Calculate the SHA-256 hash using Web Crypto API
		const hashBuffer = await crypto.subtle.digest('SHA-256', uint8Array);

		// Convert the hash to a hexadecimal string
		const hashArray = Array.from(new Uint8Array(hashBuffer));
		const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join('');

Timothy J. Baek's avatar
Timothy J. Baek committed
309
		return `${hashHex}`;
310
311
312
313
314
	} catch (error) {
		console.error('Error calculating SHA-256 hash:', error);
		throw error;
	}
};
315
316
317

export const getImportOrigin = (_chats) => {
	// Check what external service chat imports are from
Timothy J. Baek's avatar
Timothy J. Baek committed
318
319
320
321
322
	if ('mapping' in _chats[0]) {
		return 'openai';
	}
	return 'webui';
};
323

Timothy J. Baek's avatar
Timothy J. Baek committed
324
const convertOpenAIMessages = (convo) => {
325
	// Parse OpenAI chat messages and create chat dictionary for creating new chats
Timothy J. Baek's avatar
Timothy J. Baek committed
326
	const mapping = convo['mapping'];
327
	const messages = [];
Timothy J. Baek's avatar
Timothy J. Baek committed
328
	let currentId = '';
329
	let lastId = null;
330

Timothy J. Baek's avatar
Timothy J. Baek committed
331
332
	for (let message_id in mapping) {
		const message = mapping[message_id];
333
		currentId = message_id;
334
		try {
Timothy J. Baek's avatar
Timothy J. Baek committed
335
336
337
338
339
340
			if (
				messages.length == 0 &&
				(message['message'] == null ||
					(message['message']['content']['parts']?.[0] == '' &&
						message['message']['content']['text'] == null))
			) {
341
342
343
344
345
346
347
348
				// Skip chat messages with no content
				continue;
			} else {
				const new_chat = {
					id: message_id,
					parentId: lastId,
					childrenIds: message['children'] || [],
					role: message['message']?.['author']?.['role'] !== 'user' ? 'assistant' : 'user',
Timothy J. Baek's avatar
Timothy J. Baek committed
349
350
351
352
					content:
						message['message']?.['content']?.['parts']?.[0] ||
						message['message']?.['content']?.['text'] ||
						'',
353
354
355
356
357
358
359
360
					model: 'gpt-3.5-turbo',
					done: true,
					context: null
				};
				messages.push(new_chat);
				lastId = currentId;
			}
		} catch (error) {
Timothy J. Baek's avatar
Timothy J. Baek committed
361
			console.log('Error with', message, '\nError:', error);
362
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
363
	}
364
365

	let history = {};
Timothy J. Baek's avatar
Timothy J. Baek committed
366
	messages.forEach((obj) => (history[obj.id] = obj));
367
368

	const chat = {
Timothy J. Baek's avatar
Timothy J. Baek committed
369
370
371
		history: {
			currentId: currentId,
			messages: history // Need to convert this to not a list and instead a json object
372
		},
Timothy J. Baek's avatar
Timothy J. Baek committed
373
		models: ['gpt-3.5-turbo'],
Timothy J. Baek's avatar
Timothy J. Baek committed
374
375
376
		messages: messages,
		options: {},
		timestamp: convo['create_time'],
Timothy J. Baek's avatar
Timothy J. Baek committed
377
		title: convo['title'] ?? 'New Chat'
Timothy J. Baek's avatar
Timothy J. Baek committed
378
379
380
	};
	return chat;
};
381

382
383
384
385
const validateChat = (chat) => {
	// Because ChatGPT sometimes has features we can't use like DALL-E or migh have corrupted messages, need to validate
	const messages = chat.messages;

Timothy J. Baek's avatar
Timothy J. Baek committed
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
	// Check if messages array is empty
	if (messages.length === 0) {
		return false;
	}

	// Last message's children should be an empty array
	const lastMessage = messages[messages.length - 1];
	if (lastMessage.childrenIds.length !== 0) {
		return false;
	}

	// First message's parent should be null
	const firstMessage = messages[0];
	if (firstMessage.parentId !== null) {
		return false;
	}

	// Every message's content should be a string
	for (let message of messages) {
		if (typeof message.content !== 'string') {
			return false;
		}
	}

	return true;
411
412
};

Timothy J. Baek's avatar
Timothy J. Baek committed
413
export const convertOpenAIChats = (_chats) => {
414
	// Create a list of dictionaries with each conversation from import
Timothy J. Baek's avatar
Timothy J. Baek committed
415
	const chats = [];
416
	let failed = 0;
Timothy J. Baek's avatar
Timothy J. Baek committed
417
	for (let convo of _chats) {
Timothy J. Baek's avatar
Timothy J. Baek committed
418
419
		const chat = convertOpenAIMessages(convo);

420
		if (validateChat(chat)) {
Timothy J. Baek's avatar
Timothy J. Baek committed
421
422
423
424
425
426
427
			chats.push({
				id: convo['id'],
				user_id: '',
				title: convo['title'],
				chat: chat,
				timestamp: convo['timestamp']
			});
Timothy J. Baek's avatar
Timothy J. Baek committed
428
429
430
		} else {
			failed++;
		}
431
	}
Timothy J. Baek's avatar
Timothy J. Baek committed
432
	console.log(failed, 'Conversations could not be imported');
Timothy J. Baek's avatar
Timothy J. Baek committed
433
434
	return chats;
};
Timothy J. Baek's avatar
Timothy J. Baek committed
435
436
437
438
439
440
441
442
443
444
445
446

export const isValidHttpUrl = (string) => {
	let url;

	try {
		url = new URL(string);
	} catch (_) {
		return false;
	}

	return url.protocol === 'http:' || url.protocol === 'https:';
};
Timothy J. Baek's avatar
Timothy J. Baek committed
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463

export const removeEmojis = (str) => {
	// Regular expression to match emojis
	const emojiRegex = /[\uD800-\uDBFF][\uDC00-\uDFFF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDE4F]/g;

	// Replace emojis with an empty string
	return str.replace(emojiRegex, '');
};

export const extractSentences = (text) => {
	// Split the paragraph into sentences based on common punctuation marks
	const sentences = text.split(/(?<=[.!?])/);

	return sentences
		.map((sentence) => removeEmojis(sentence.trim()))
		.filter((sentence) => sentence !== '');
};
Timothy J. Baek's avatar
Timothy J. Baek committed
464
465
466
467
468
469

export const blobToFile = (blob, fileName) => {
	// Create a new File object from the Blob
	const file = new File([blob], fileName, { type: blob.type });
	return file;
};