feat: make chunk splitting a configurable option

src/lib/apis/streaming/index.ts

@@ -6,9 +6,14 @@ type TextStreamUpdate = {
 // createOpenAITextStream takes a ReadableStreamDefaultReader from an SSE response,
 // and returns an async generator that emits delta updates with large deltas chunked into random sized chunks
 export async function createOpenAITextStream(
-	messageStream: ReadableStreamDefaultReader
+	messageStream: ReadableStreamDefaultReader,
+	splitLargeDeltas: boolean
 ): Promise<AsyncGenerator<TextStreamUpdate>> {
-	return streamLargeDeltasAsRandomChunks(openAIStreamToIterator(messageStream));
+	let iterator = openAIStreamToIterator(messageStream);
+	if (splitLargeDeltas) {
+		iterator = streamLargeDeltasAsRandomChunks(iterator);
+	}
+	return iterator;
 }
 
 async function* openAIStreamToIterator(

src/lib/components/chat/Settings/Interface.svelte

@@ -17,11 +17,17 @@
 	let titleAutoGenerateModelExternal = '';
 	let fullScreenMode = false;
 	let titleGenerationPrompt = '';
+	let splitLargeChunks = false;
 
 	// Interface
 	let promptSuggestions = [];
 	let showUsername = false;
 
+	const toggleSplitLargeChunks = async () => {
+		splitLargeChunks = !splitLargeChunks;
+		saveSettings({ splitLargeChunks: splitLargeChunks });
+	};
+
 	const toggleFullScreenMode = async () => {
 		fullScreenMode = !fullScreenMode;
 		saveSettings({ fullScreenMode: fullScreenMode });
@@ -197,6 +203,28 @@
 					</button>
 				</div>
 			</div>
+
+			<div>
+				<div class=" py-0.5 flex w-full justify-between">
+					<div class=" self-center text-xs font-medium">
+						{$i18n.t('Fluidly stream large external response chunks')}
+					</div>
+
+					<button
+						class="p-1 px-3 text-xs flex rounded transition"
+						on:click={() => {
+							toggleSplitLargeChunks();
+						}}
+						type="button"
+					>
+						{#if splitLargeChunks === true}
+							<span class="ml-2 self-center">{$i18n.t('On')}</span>
+						{:else}
+							<span class="ml-2 self-center">{$i18n.t('Off')}</span>
+						{/if}
+					</button>
+				</div>
+			</div>
 		</div>
 
 		<hr class=" dark:border-gray-700" />

src/lib/i18n/locales/en-US/translation.json

@@ -152,6 +152,7 @@
 	"File Mode": "",
 	"File not found.": "",
 	"Fingerprint spoofing detected: Unable to use initials as avatar. Defaulting to default profile image.": "",
+	"Fluidly stream large external response chunks": "",
 	"Focus chat input": "",
 	"Format your variables using square brackets like this:": "",
 	"From (Base Model)": "",

src/routes/(app)/+page.svelte

@@ -600,7 +600,7 @@
 				.pipeThrough(splitStream('\n'))
 				.getReader();
 
-			const textStream = await createOpenAITextStream(reader);
+			const textStream = await createOpenAITextStream(reader, $settings.splitLargeChunks);
 			console.log(textStream);
 
 			for await (const update of textStream) {

src/routes/(app)/c/[id]/+page.svelte

@@ -552,9 +552,9 @@
 				messages: [
 					$settings.system
 						? {
-							role: 'system',
-							content: $settings.system
-						}
+								role: 'system',
+								content: $settings.system
+						  }
 						: undefined,
 					...messages
 				]
@@ -612,7 +612,7 @@
 				.pipeThrough(splitStream('\n'))
 				.getReader();
 
-			const textStream = await createOpenAITextStream(reader);
+			const textStream = await createOpenAITextStream(reader, $settings.splitLargeChunks);
 			console.log(textStream);
 
 			for await (const update of textStream) {