index.ts 9.16 KB
Newer Older
1
2
import { v4 as uuidv4 } from 'uuid';
import sha256 from 'js-sha256';
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import { getOllamaModels } from '$lib/apis/ollama';
import { getOpenAIModels } from '$lib/apis/openai';
import { getLiteLLMModels } from '$lib/apis/litellm';

export const getModels = async (token: string) => {
	let models = await Promise.all([
		await getOllamaModels(token).catch((error) => {
			console.log(error);
			return null;
		}),
		await getOpenAIModels(token).catch((error) => {
			console.log(error);
			return null;
		}),
		await getLiteLLMModels(token).catch((error) => {
			console.log(error);
			return null;
		})
	]);

	models = models
		.filter((models) => models)
		.reduce((a, e, i, arr) => a.concat(e, ...(i < arr.length - 1 ? [{ name: 'hr' }] : [])), []);

	return models;
};
29
30
31
32
33

//////////////////////////
// Helper functions
//////////////////////////

Timothy J. Baek's avatar
Timothy J. Baek committed
34
35
36
37
export const capitalizeFirstLetter = (string) => {
	return string.charAt(0).toUpperCase() + string.slice(1);
};

38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
export const splitStream = (splitOn) => {
	let buffer = '';
	return new TransformStream({
		transform(chunk, controller) {
			buffer += chunk;
			const parts = buffer.split(splitOn);
			parts.slice(0, -1).forEach((part) => controller.enqueue(part));
			buffer = parts[parts.length - 1];
		},
		flush(controller) {
			if (buffer) controller.enqueue(buffer);
		}
	});
};

export const convertMessagesToHistory = (messages) => {
54
	const history = {
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
		messages: {},
		currentId: null
	};

	let parentMessageId = null;
	let messageId = null;

	for (const message of messages) {
		messageId = uuidv4();

		if (parentMessageId !== null) {
			history.messages[parentMessageId].childrenIds = [
				...history.messages[parentMessageId].childrenIds,
				messageId
			];
		}

		history.messages[messageId] = {
			...message,
			id: messageId,
			parentId: parentMessageId,
			childrenIds: []
		};

		parentMessageId = messageId;
	}

	history.currentId = messageId;
	return history;
};

export const getGravatarURL = (email) => {
	// Trim leading and trailing whitespace from
	// an email address and force all characters
	// to lower case
	const address = String(email).trim().toLowerCase();

	// Create a SHA256 hash of the final string
	const hash = sha256(address);

	// Grab the actual image URL
	return `https://www.gravatar.com/avatar/${hash}`;
};
Timothy J. Baek's avatar
Timothy J. Baek committed
98

Timothy J. Baek's avatar
Timothy J. Baek committed
99
export const copyToClipboard = (text) => {
Timothy J. Baek's avatar
Timothy J. Baek committed
100
	if (!navigator.clipboard) {
Timothy J. Baek's avatar
Timothy J. Baek committed
101
		const textArea = document.createElement('textarea');
Timothy J. Baek's avatar
Timothy J. Baek committed
102
103
104
105
106
107
108
109
110
111
112
113
		textArea.value = text;

		// Avoid scrolling to bottom
		textArea.style.top = '0';
		textArea.style.left = '0';
		textArea.style.position = 'fixed';

		document.body.appendChild(textArea);
		textArea.focus();
		textArea.select();

		try {
Timothy J. Baek's avatar
Timothy J. Baek committed
114
115
			const successful = document.execCommand('copy');
			const msg = successful ? 'successful' : 'unsuccessful';
Timothy J. Baek's avatar
Timothy J. Baek committed
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
			console.log('Fallback: Copying text command was ' + msg);
		} catch (err) {
			console.error('Fallback: Oops, unable to copy', err);
		}

		document.body.removeChild(textArea);
		return;
	}
	navigator.clipboard.writeText(text).then(
		function () {
			console.log('Async: Copying to clipboard was successful!');
		},
		function (err) {
			console.error('Async: Could not copy text: ', err);
		}
	);
};
Timothy J. Baek's avatar
Timothy J. Baek committed
133

Timothy J. Baek's avatar
Timothy J. Baek committed
134
export const compareVersion = (latest, current) => {
135
	return current === '0.0.0'
Timothy J. Baek's avatar
Timothy J. Baek committed
136
		? false
Timothy J. Baek's avatar
Timothy J. Baek committed
137
		: current.localeCompare(latest, undefined, {
138
139
140
141
				numeric: true,
				sensitivity: 'case',
				caseFirst: 'upper'
		  }) < 0;
Timothy J. Baek's avatar
Timothy J. Baek committed
142
};
Timothy J. Baek's avatar
Timothy J. Baek committed
143
144
145

export const findWordIndices = (text) => {
	const regex = /\[([^\]]+)\]/g;
146
	const matches = [];
Timothy J. Baek's avatar
Timothy J. Baek committed
147
148
149
150
151
152
153
154
155
156
157
158
	let match;

	while ((match = regex.exec(text)) !== null) {
		matches.push({
			word: match[1],
			startIndex: match.index,
			endIndex: regex.lastIndex - 1
		});
	}

	return matches;
};
159

Timothy J. Baek's avatar
Timothy J. Baek committed
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
export const removeFirstHashWord = (inputString) => {
	// Split the string into an array of words
	const words = inputString.split(' ');

	// Find the index of the first word that starts with #
	const index = words.findIndex((word) => word.startsWith('#'));

	// Remove the first word with #
	if (index !== -1) {
		words.splice(index, 1);
	}

	// Join the remaining words back into a string
	const resultString = words.join(' ');

	return resultString;
};

178
179
180
181
182
183
184
185
186
187
188
189
190
export const transformFileName = (fileName) => {
	// Convert to lowercase
	const lowerCaseFileName = fileName.toLowerCase();

	// Remove special characters using regular expression
	const sanitizedFileName = lowerCaseFileName.replace(/[^\w\s]/g, '');

	// Replace spaces with dashes
	const finalFileName = sanitizedFileName.replace(/\s+/g, '-');

	return finalFileName;
};

191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
export const calculateSHA256 = async (file) => {
	// Create a FileReader to read the file asynchronously
	const reader = new FileReader();

	// Define a promise to handle the file reading
	const readFile = new Promise((resolve, reject) => {
		reader.onload = () => resolve(reader.result);
		reader.onerror = reject;
	});

	// Read the file as an ArrayBuffer
	reader.readAsArrayBuffer(file);

	try {
		// Wait for the FileReader to finish reading the file
		const buffer = await readFile;

		// Convert the ArrayBuffer to a Uint8Array
		const uint8Array = new Uint8Array(buffer);

		// Calculate the SHA-256 hash using Web Crypto API
		const hashBuffer = await crypto.subtle.digest('SHA-256', uint8Array);

		// Convert the hash to a hexadecimal string
		const hashArray = Array.from(new Uint8Array(hashBuffer));
		const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join('');

Timothy J. Baek's avatar
Timothy J. Baek committed
218
		return `${hashHex}`;
219
220
221
222
223
	} catch (error) {
		console.error('Error calculating SHA-256 hash:', error);
		throw error;
	}
};
224
225
226

export const getImportOrigin = (_chats) => {
	// Check what external service chat imports are from
Timothy J. Baek's avatar
Timothy J. Baek committed
227
228
229
230
231
	if ('mapping' in _chats[0]) {
		return 'openai';
	}
	return 'webui';
};
232

Timothy J. Baek's avatar
Timothy J. Baek committed
233
const convertOpenAIMessages = (convo) => {
234
	// Parse OpenAI chat messages and create chat dictionary for creating new chats
Timothy J. Baek's avatar
Timothy J. Baek committed
235
	const mapping = convo['mapping'];
236
	const messages = [];
Timothy J. Baek's avatar
Timothy J. Baek committed
237
	let currentId = '';
238
	let lastId = null;
239

Timothy J. Baek's avatar
Timothy J. Baek committed
240
241
	for (let message_id in mapping) {
		const message = mapping[message_id];
242
		currentId = message_id;
243
		try {
Timothy J. Baek's avatar
Timothy J. Baek committed
244
245
246
247
248
249
			if (
				messages.length == 0 &&
				(message['message'] == null ||
					(message['message']['content']['parts']?.[0] == '' &&
						message['message']['content']['text'] == null))
			) {
250
251
252
253
254
255
256
257
				// Skip chat messages with no content
				continue;
			} else {
				const new_chat = {
					id: message_id,
					parentId: lastId,
					childrenIds: message['children'] || [],
					role: message['message']?.['author']?.['role'] !== 'user' ? 'assistant' : 'user',
Timothy J. Baek's avatar
Timothy J. Baek committed
258
259
260
261
					content:
						message['message']?.['content']?.['parts']?.[0] ||
						message['message']?.['content']?.['text'] ||
						'',
262
263
264
265
266
267
268
269
					model: 'gpt-3.5-turbo',
					done: true,
					context: null
				};
				messages.push(new_chat);
				lastId = currentId;
			}
		} catch (error) {
Timothy J. Baek's avatar
Timothy J. Baek committed
270
			console.log('Error with', message, '\nError:', error);
271
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
272
	}
273
274

	let history = {};
Timothy J. Baek's avatar
Timothy J. Baek committed
275
	messages.forEach((obj) => (history[obj.id] = obj));
276
277

	const chat = {
Timothy J. Baek's avatar
Timothy J. Baek committed
278
279
280
		history: {
			currentId: currentId,
			messages: history // Need to convert this to not a list and instead a json object
281
		},
Timothy J. Baek's avatar
Timothy J. Baek committed
282
		models: ['gpt-3.5-turbo'],
Timothy J. Baek's avatar
Timothy J. Baek committed
283
284
285
		messages: messages,
		options: {},
		timestamp: convo['create_time'],
Timothy J. Baek's avatar
Timothy J. Baek committed
286
		title: convo['title'] ?? 'New Chat'
Timothy J. Baek's avatar
Timothy J. Baek committed
287
288
289
	};
	return chat;
};
290

291
292
293
294
const validateChat = (chat) => {
	// Because ChatGPT sometimes has features we can't use like DALL-E or migh have corrupted messages, need to validate
	const messages = chat.messages;

Timothy J. Baek's avatar
Timothy J. Baek committed
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
	// Check if messages array is empty
	if (messages.length === 0) {
		return false;
	}

	// Last message's children should be an empty array
	const lastMessage = messages[messages.length - 1];
	if (lastMessage.childrenIds.length !== 0) {
		return false;
	}

	// First message's parent should be null
	const firstMessage = messages[0];
	if (firstMessage.parentId !== null) {
		return false;
	}

	// Every message's content should be a string
	for (let message of messages) {
		if (typeof message.content !== 'string') {
			return false;
		}
	}

	return true;
320
321
};

Timothy J. Baek's avatar
Timothy J. Baek committed
322
export const convertOpenAIChats = (_chats) => {
323
	// Create a list of dictionaries with each conversation from import
Timothy J. Baek's avatar
Timothy J. Baek committed
324
	const chats = [];
325
	let failed = 0;
Timothy J. Baek's avatar
Timothy J. Baek committed
326
	for (let convo of _chats) {
Timothy J. Baek's avatar
Timothy J. Baek committed
327
328
		const chat = convertOpenAIMessages(convo);

329
		if (validateChat(chat)) {
Timothy J. Baek's avatar
Timothy J. Baek committed
330
331
332
333
334
335
336
			chats.push({
				id: convo['id'],
				user_id: '',
				title: convo['title'],
				chat: chat,
				timestamp: convo['timestamp']
			});
Timothy J. Baek's avatar
Timothy J. Baek committed
337
338
339
		} else {
			failed++;
		}
340
	}
Timothy J. Baek's avatar
Timothy J. Baek committed
341
	console.log(failed, 'Conversations could not be imported');
Timothy J. Baek's avatar
Timothy J. Baek committed
342
343
	return chats;
};
Timothy J. Baek's avatar
Timothy J. Baek committed
344
345
346
347
348
349
350
351
352
353
354
355

export const isValidHttpUrl = (string) => {
	let url;

	try {
		url = new URL(string);
	} catch (_) {
		return false;
	}

	return url.protocol === 'http:' || url.protocol === 'https:';
};
Timothy J. Baek's avatar
Timothy J. Baek committed
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372

export const removeEmojis = (str) => {
	// Regular expression to match emojis
	const emojiRegex = /[\uD800-\uDBFF][\uDC00-\uDFFF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDE4F]/g;

	// Replace emojis with an empty string
	return str.replace(emojiRegex, '');
};

export const extractSentences = (text) => {
	// Split the paragraph into sentences based on common punctuation marks
	const sentences = text.split(/(?<=[.!?])/);

	return sentences
		.map((sentence) => removeEmojis(sentence.trim()))
		.filter((sentence) => sentence !== '');
};
Timothy J. Baek's avatar
Timothy J. Baek committed
373
374
375
376
377
378

export const blobToFile = (blob, fileName) => {
	// Create a new File object from the Blob
	const file = new File([blob], fileName, { type: blob.type });
	return file;
};