index.ts 9.05 KB
Newer Older
1
2
import { v4 as uuidv4 } from 'uuid';
import sha256 from 'js-sha256';
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import { getOllamaModels } from '$lib/apis/ollama';
import { getOpenAIModels } from '$lib/apis/openai';
import { getLiteLLMModels } from '$lib/apis/litellm';

export const getModels = async (token: string) => {
	let models = await Promise.all([
		await getOllamaModels(token).catch((error) => {
			console.log(error);
			return null;
		}),
		await getOpenAIModels(token).catch((error) => {
			console.log(error);
			return null;
		}),
		await getLiteLLMModels(token).catch((error) => {
			console.log(error);
			return null;
		})
	]);

	models = models
		.filter((models) => models)
		.reduce((a, e, i, arr) => a.concat(e, ...(i < arr.length - 1 ? [{ name: 'hr' }] : [])), []);

	return models;
};
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

//////////////////////////
// Helper functions
//////////////////////////

export const splitStream = (splitOn) => {
	let buffer = '';
	return new TransformStream({
		transform(chunk, controller) {
			buffer += chunk;
			const parts = buffer.split(splitOn);
			parts.slice(0, -1).forEach((part) => controller.enqueue(part));
			buffer = parts[parts.length - 1];
		},
		flush(controller) {
			if (buffer) controller.enqueue(buffer);
		}
	});
};

export const convertMessagesToHistory = (messages) => {
50
	const history = {
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
		messages: {},
		currentId: null
	};

	let parentMessageId = null;
	let messageId = null;

	for (const message of messages) {
		messageId = uuidv4();

		if (parentMessageId !== null) {
			history.messages[parentMessageId].childrenIds = [
				...history.messages[parentMessageId].childrenIds,
				messageId
			];
		}

		history.messages[messageId] = {
			...message,
			id: messageId,
			parentId: parentMessageId,
			childrenIds: []
		};

		parentMessageId = messageId;
	}

	history.currentId = messageId;
	return history;
};

export const getGravatarURL = (email) => {
	// Trim leading and trailing whitespace from
	// an email address and force all characters
	// to lower case
	const address = String(email).trim().toLowerCase();

	// Create a SHA256 hash of the final string
	const hash = sha256(address);

	// Grab the actual image URL
	return `https://www.gravatar.com/avatar/${hash}`;
};
Timothy J. Baek's avatar
Timothy J. Baek committed
94

Timothy J. Baek's avatar
Timothy J. Baek committed
95
export const copyToClipboard = (text) => {
Timothy J. Baek's avatar
Timothy J. Baek committed
96
	if (!navigator.clipboard) {
Timothy J. Baek's avatar
Timothy J. Baek committed
97
		const textArea = document.createElement('textarea');
Timothy J. Baek's avatar
Timothy J. Baek committed
98
99
100
101
102
103
104
105
106
107
108
109
		textArea.value = text;

		// Avoid scrolling to bottom
		textArea.style.top = '0';
		textArea.style.left = '0';
		textArea.style.position = 'fixed';

		document.body.appendChild(textArea);
		textArea.focus();
		textArea.select();

		try {
Timothy J. Baek's avatar
Timothy J. Baek committed
110
111
			const successful = document.execCommand('copy');
			const msg = successful ? 'successful' : 'unsuccessful';
Timothy J. Baek's avatar
Timothy J. Baek committed
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
			console.log('Fallback: Copying text command was ' + msg);
		} catch (err) {
			console.error('Fallback: Oops, unable to copy', err);
		}

		document.body.removeChild(textArea);
		return;
	}
	navigator.clipboard.writeText(text).then(
		function () {
			console.log('Async: Copying to clipboard was successful!');
		},
		function (err) {
			console.error('Async: Could not copy text: ', err);
		}
	);
};
Timothy J. Baek's avatar
Timothy J. Baek committed
129

Timothy J. Baek's avatar
Timothy J. Baek committed
130
export const compareVersion = (latest, current) => {
131
	return current === '0.0.0'
Timothy J. Baek's avatar
Timothy J. Baek committed
132
		? false
Timothy J. Baek's avatar
Timothy J. Baek committed
133
		: current.localeCompare(latest, undefined, {
134
135
136
137
				numeric: true,
				sensitivity: 'case',
				caseFirst: 'upper'
		  }) < 0;
Timothy J. Baek's avatar
Timothy J. Baek committed
138
};
Timothy J. Baek's avatar
Timothy J. Baek committed
139
140
141

export const findWordIndices = (text) => {
	const regex = /\[([^\]]+)\]/g;
142
	const matches = [];
Timothy J. Baek's avatar
Timothy J. Baek committed
143
144
145
146
147
148
149
150
151
152
153
154
	let match;

	while ((match = regex.exec(text)) !== null) {
		matches.push({
			word: match[1],
			startIndex: match.index,
			endIndex: regex.lastIndex - 1
		});
	}

	return matches;
};
155

Timothy J. Baek's avatar
Timothy J. Baek committed
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
export const removeFirstHashWord = (inputString) => {
	// Split the string into an array of words
	const words = inputString.split(' ');

	// Find the index of the first word that starts with #
	const index = words.findIndex((word) => word.startsWith('#'));

	// Remove the first word with #
	if (index !== -1) {
		words.splice(index, 1);
	}

	// Join the remaining words back into a string
	const resultString = words.join(' ');

	return resultString;
};

174
175
176
177
178
179
180
181
182
183
184
185
186
export const transformFileName = (fileName) => {
	// Convert to lowercase
	const lowerCaseFileName = fileName.toLowerCase();

	// Remove special characters using regular expression
	const sanitizedFileName = lowerCaseFileName.replace(/[^\w\s]/g, '');

	// Replace spaces with dashes
	const finalFileName = sanitizedFileName.replace(/\s+/g, '-');

	return finalFileName;
};

187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
export const calculateSHA256 = async (file) => {
	// Create a FileReader to read the file asynchronously
	const reader = new FileReader();

	// Define a promise to handle the file reading
	const readFile = new Promise((resolve, reject) => {
		reader.onload = () => resolve(reader.result);
		reader.onerror = reject;
	});

	// Read the file as an ArrayBuffer
	reader.readAsArrayBuffer(file);

	try {
		// Wait for the FileReader to finish reading the file
		const buffer = await readFile;

		// Convert the ArrayBuffer to a Uint8Array
		const uint8Array = new Uint8Array(buffer);

		// Calculate the SHA-256 hash using Web Crypto API
		const hashBuffer = await crypto.subtle.digest('SHA-256', uint8Array);

		// Convert the hash to a hexadecimal string
		const hashArray = Array.from(new Uint8Array(hashBuffer));
		const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join('');

Timothy J. Baek's avatar
Timothy J. Baek committed
214
		return `${hashHex}`;
215
216
217
218
219
	} catch (error) {
		console.error('Error calculating SHA-256 hash:', error);
		throw error;
	}
};
220
221
222

export const getImportOrigin = (_chats) => {
	// Check what external service chat imports are from
Timothy J. Baek's avatar
Timothy J. Baek committed
223
224
225
226
227
	if ('mapping' in _chats[0]) {
		return 'openai';
	}
	return 'webui';
};
228

Timothy J. Baek's avatar
Timothy J. Baek committed
229
const convertOpenAIMessages = (convo) => {
230
	// Parse OpenAI chat messages and create chat dictionary for creating new chats
Timothy J. Baek's avatar
Timothy J. Baek committed
231
	const mapping = convo['mapping'];
232
	const messages = [];
Timothy J. Baek's avatar
Timothy J. Baek committed
233
	let currentId = '';
234
	let lastId = null;
235

Timothy J. Baek's avatar
Timothy J. Baek committed
236
237
	for (let message_id in mapping) {
		const message = mapping[message_id];
238
		currentId = message_id;
239
		try {
Timothy J. Baek's avatar
Timothy J. Baek committed
240
241
242
243
244
245
			if (
				messages.length == 0 &&
				(message['message'] == null ||
					(message['message']['content']['parts']?.[0] == '' &&
						message['message']['content']['text'] == null))
			) {
246
247
248
249
250
251
252
253
				// Skip chat messages with no content
				continue;
			} else {
				const new_chat = {
					id: message_id,
					parentId: lastId,
					childrenIds: message['children'] || [],
					role: message['message']?.['author']?.['role'] !== 'user' ? 'assistant' : 'user',
Timothy J. Baek's avatar
Timothy J. Baek committed
254
255
256
257
					content:
						message['message']?.['content']?.['parts']?.[0] ||
						message['message']?.['content']?.['text'] ||
						'',
258
259
260
261
262
263
264
265
					model: 'gpt-3.5-turbo',
					done: true,
					context: null
				};
				messages.push(new_chat);
				lastId = currentId;
			}
		} catch (error) {
Timothy J. Baek's avatar
Timothy J. Baek committed
266
			console.log('Error with', message, '\nError:', error);
267
		}
Timothy J. Baek's avatar
Timothy J. Baek committed
268
	}
269
270

	let history = {};
Timothy J. Baek's avatar
Timothy J. Baek committed
271
	messages.forEach((obj) => (history[obj.id] = obj));
272
273

	const chat = {
Timothy J. Baek's avatar
Timothy J. Baek committed
274
275
276
		history: {
			currentId: currentId,
			messages: history // Need to convert this to not a list and instead a json object
277
		},
Timothy J. Baek's avatar
Timothy J. Baek committed
278
		models: ['gpt-3.5-turbo'],
Timothy J. Baek's avatar
Timothy J. Baek committed
279
280
281
		messages: messages,
		options: {},
		timestamp: convo['create_time'],
Timothy J. Baek's avatar
Timothy J. Baek committed
282
		title: convo['title'] ?? 'New Chat'
Timothy J. Baek's avatar
Timothy J. Baek committed
283
284
285
	};
	return chat;
};
286

287
288
289
290
const validateChat = (chat) => {
	// Because ChatGPT sometimes has features we can't use like DALL-E or migh have corrupted messages, need to validate
	const messages = chat.messages;

Timothy J. Baek's avatar
Timothy J. Baek committed
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
	// Check if messages array is empty
	if (messages.length === 0) {
		return false;
	}

	// Last message's children should be an empty array
	const lastMessage = messages[messages.length - 1];
	if (lastMessage.childrenIds.length !== 0) {
		return false;
	}

	// First message's parent should be null
	const firstMessage = messages[0];
	if (firstMessage.parentId !== null) {
		return false;
	}

	// Every message's content should be a string
	for (let message of messages) {
		if (typeof message.content !== 'string') {
			return false;
		}
	}

	return true;
316
317
};

Timothy J. Baek's avatar
Timothy J. Baek committed
318
export const convertOpenAIChats = (_chats) => {
319
	// Create a list of dictionaries with each conversation from import
Timothy J. Baek's avatar
Timothy J. Baek committed
320
	const chats = [];
321
	let failed = 0;
Timothy J. Baek's avatar
Timothy J. Baek committed
322
	for (let convo of _chats) {
Timothy J. Baek's avatar
Timothy J. Baek committed
323
324
		const chat = convertOpenAIMessages(convo);

325
		if (validateChat(chat)) {
Timothy J. Baek's avatar
Timothy J. Baek committed
326
327
328
329
330
331
332
			chats.push({
				id: convo['id'],
				user_id: '',
				title: convo['title'],
				chat: chat,
				timestamp: convo['timestamp']
			});
Timothy J. Baek's avatar
Timothy J. Baek committed
333
334
335
		} else {
			failed++;
		}
336
	}
Timothy J. Baek's avatar
Timothy J. Baek committed
337
	console.log(failed, 'Conversations could not be imported');
Timothy J. Baek's avatar
Timothy J. Baek committed
338
339
	return chats;
};
Timothy J. Baek's avatar
Timothy J. Baek committed
340
341
342
343
344
345
346
347
348
349
350
351

export const isValidHttpUrl = (string) => {
	let url;

	try {
		url = new URL(string);
	} catch (_) {
		return false;
	}

	return url.protocol === 'http:' || url.protocol === 'https:';
};
Timothy J. Baek's avatar
Timothy J. Baek committed
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368

export const removeEmojis = (str) => {
	// Regular expression to match emojis
	const emojiRegex = /[\uD800-\uDBFF][\uDC00-\uDFFF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDE4F]/g;

	// Replace emojis with an empty string
	return str.replace(emojiRegex, '');
};

export const extractSentences = (text) => {
	// Split the paragraph into sentences based on common punctuation marks
	const sentences = text.split(/(?<=[.!?])/);

	return sentences
		.map((sentence) => removeEmojis(sentence.trim()))
		.filter((sentence) => sentence !== '');
};
Timothy J. Baek's avatar
Timothy J. Baek committed
369
370
371
372
373
374

export const blobToFile = (blob, fileName) => {
	// Create a new File object from the Blob
	const file = new File([blob], fileName, { type: blob.type });
	return file;
};