revert: markdown rendering

src/lib/components/chat/Messages/MarkdownInlineTokens.svelte

 <script lang="ts">
 	import type { Token } from 'marked';
 	import { unescapeHtml } from '$lib/utils';
-	import { onMount } from 'svelte';
-	import { revertSanitizedResponseContent } from '$lib/utils/index.js';
 	import Image from '$lib/components/common/Image.svelte';
 
 	export let id: string;
 	export let tokens: Token[];
-
-	onMount(() => {
-		console.log('MarkdownInlineTokens', id, tokens, top);
-	});
 </script>
 
 {#each tokens as token}

src/lib/components/chat/Messages/MarkdownTokens.svelte

 <script lang="ts">
+	import { marked } from 'marked';
 	import type { Token } from 'marked';
 	import { revertSanitizedResponseContent, unescapeHtml } from '$lib/utils';
 	import CodeBlock from '$lib/components/chat/Messages/CodeBlock.svelte';
@@ -13,57 +14,55 @@
 		return 'h' + depth;
 	};
 
-	onMount(() => {
-		console.log('MarkdownTokens', id, tokens, top);
-	});
+	const renderer = new marked.Renderer();
+	// For code blocks with simple backticks
+	renderer.codespan = (code) => {
+		return `<code class="codespan">${code.replaceAll('&amp;', '&')}</code>`;
+	};
+
+	renderer.code = (code, lang) => {
+		return `<pre><code class="language-${lang}">${code}</code></pre>`;
+	};
+
+	// Open all links in a new tab/window (from https://github.com/markedjs/marked/issues/655#issuecomment-383226346)
+	const origLinkRenderer = renderer.link;
+	renderer.link = (href, title, text) => {
+		const html = origLinkRenderer.call(renderer, href, title, text);
+		return html.replace(/^<a /, '<a target="_blank" rel="nofollow" ');
+	};
+	const { extensions, ...defaults } = marked.getDefaults() as marked.MarkedOptions & {
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		extensions: any;
+	};
 </script>
 
-{#each tokens as token, tokenIdx}
-	{#if token.type === 'hr'}
-		<hr />
-	{:else if token.type === 'heading'}
+{#each tokens as token, tokenIdx (`${id}-${tokenIdx}`)}
+	{#if token.type === 'code'}
+		{#if token.lang === 'mermaid'}
+			<pre class="mermaid">{revertSanitizedResponseContent(token.text)}</pre>
+		{:else}
+			<CodeBlock
+				id={`${id}-${tokenIdx}`}
+				lang={token?.lang ?? ''}
+				code={revertSanitizedResponseContent(token?.text ?? '')}
+			/>
+		{/if}
+		<!-- {:else if token.type === 'heading'}
 		<svelte:element this={headerComponent(token.depth)}>
 			<MarkdownInlineTokens id={`${id}-${tokenIdx}-h`} tokens={token.tokens} />
 		</svelte:element>
-	{:else if token.type === 'code'}
-		<CodeBlock
-			{id}
-			lang={token?.lang ?? ''}
-			code={revertSanitizedResponseContent(token?.text ?? '')}
-		/>
-	{:else if token.type === 'table'}
-		<table>
-			<thead>
-				<tr>
-					{#each token.header as header, headerIdx}
-						<th style={token.align[headerIdx] ? '' : `text-align: ${token.align[headerIdx]}`}>
-							<MarkdownInlineTokens
-								id={`${id}-${tokenIdx}-header-${headerIdx}`}
-								tokens={header.tokens}
-							/>
-						</th>
-					{/each}
-				</tr>
-			</thead>
-			<tbody>
-				{#each token.rows as row, rowIdx}
-					<tr>
-						{#each row ?? [] as cell, cellIdx}
-							<td style={token.align[cellIdx] ? '' : `text-align: ${token.align[cellIdx]}`}>
-								<MarkdownInlineTokens
-									id={`${id}-${tokenIdx}-row-${rowIdx}-${cellIdx}`}
-									tokens={cell.tokens}
-								/>
-							</td>
-						{/each}
-					</tr>
-				{/each}
-			</tbody>
-		</table>
+	{:else if token.type === 'hr'}
+		<hr />
 	{:else if token.type === 'blockquote'}
 		<blockquote>
 			<svelte:self id={`${id}-${tokenIdx}`} tokens={token.tokens} />
 		</blockquote>
+	{:else if token.type === 'html'}
+		{@html token.text}
+	{:else if token.type === 'paragraph'}
+		<p>
+			<MarkdownInlineTokens id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} />
+		</p>
 	{:else if token.type === 'list'}
 		{#if token.ordered}
 			<ol start={token.start || 1}>
@@ -90,14 +89,37 @@
 				{/each}
 			</ul>
 		{/if}
-	{:else if token.type === 'html'}
-		{@html token.text}
-	{:else if token.type === 'paragraph'}
-		<p>
-			<MarkdownInlineTokens id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} />
-		</p>
-	{:else if token.type === 'text'}
-		{#if top}
+	{:else if token.type === 'table'}
+		<table>
+			<thead>
+				<tr>
+					{#each token.header as header, headerIdx}
+						<th style={token.align[headerIdx] ? '' : `text-align: ${token.align[headerIdx]}`}>
+							<MarkdownInlineTokens
+								id={`${id}-${tokenIdx}-header-${headerIdx}`}
+								tokens={header.tokens}
+							/>
+						</th>
+					{/each}
+				</tr>
+			</thead>
+			<tbody>
+				{#each token.rows as row, rowIdx}
+					<tr>
+						{#each row ?? [] as cell, cellIdx}
+							<td style={token.align[cellIdx] ? '' : `text-align: ${token.align[cellIdx]}`}>
+								<MarkdownInlineTokens
+									id={`${id}-${tokenIdx}-row-${rowIdx}-${cellIdx}`}
+									tokens={cell.tokens}
+								/>
+							</td>
+						{/each}
+					</tr>
+				{/each}
+			</tbody>
+		</table>
+	{:else if token.type === 'text'} -->
+		<!-- {#if top}
 			<p>
 				{#if token.tokens}
 					<MarkdownInlineTokens id={`${id}-${tokenIdx}-t`} tokens={token.tokens} />
@@ -109,10 +131,13 @@
 			<MarkdownInlineTokens id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} />
 		{:else}
 			{unescapeHtml(token.text)}
-		{/if}
-	{:else if token.type === 'space'}
-		{''}
+		{/if} -->
 	{:else}
-		{console.log('Unknown token', token)}
+		{@html marked.parse(token.raw, {
+			...defaults,
+			gfm: true,
+			breaks: true,
+			renderer
+		})}
 	{/if}
 {/each}

src/lib/components/chat/Messages/ResponseMessage.svelte

@@ -77,9 +77,16 @@
 
 	let selectedCitation = null;
 
-	$: tokens = marked.lexer(
-		replaceTokens(sanitizeResponseContent(message?.content), model?.name, $user?.name)
-	);
+	let tokens;
+
+	$: (async () => {
+		if (message?.content) {
+			tokens = marked.lexer(
+				replaceTokens(sanitizeResponseContent(message?.content), model?.name, $user?.name)
+			);
+			// console.log(message?.content, tokens);
+		}
+	})();
 
 	$: if (message) {
 		renderStyling();
@@ -413,7 +420,7 @@
 				{/if}
 
 				<div
-					class="prose chat-{message.role} w-full max-w-full dark:prose-invert prose-p:my-0 prose-img:my-1 prose-headings:my-1.5 prose-ol:my-1 prose-ul:my-1 whitespace-pre-line"
+					class="prose chat-{message.role} w-full max-w-full dark:prose-invert prose-p:my-0 prose-img:my-1 prose-headings:my-1.5 prose-ol:my-1 prose-ul:my-1 prose-li:my-0 whitespace-pre-line"
 				>
 					<div>
 						{#if (message?.statusHistory ?? [...(message?.status ? [message?.status] : [])]).length > 0}
@@ -493,14 +500,13 @@
 								</div>
 							</div>
 						{:else}
-							<div class="w-full">
+							<div class="w-full flex flex-col">
 								{#if message.content === '' && !message.error}
 									<Skeleton />
 								{:else if message.content && message.error !== true}
 									<!-- always show message contents even if there's an error -->
 									<!-- unless message.error === true which is legacy error handling, where the error message is stored in message.content -->
-
-									{#key tokens}
+									{#key message.id}
 										<MarkdownTokens id={message.id} {tokens} />
 									{/key}
 								{/if}