Merge pull request #3161 from sammcj/num_batch

feat: add num_keep, num_batch

backend/apps/ollama/main.py

@@ -751,6 +751,14 @@ async def generate_chat_completion(
             if model_info.params.get("num_ctx", None):
                 payload["options"]["num_ctx"] = model_info.params.get("num_ctx", None)
 
+            if model_info.params.get("num_batch", None):
+                payload["options"]["num_batch"] = model_info.params.get(
+                    "num_batch", None
+                )
+
+            if model_info.params.get("num_keep", None):
+                payload["options"]["num_keep"] = model_info.params.get("num_keep", None)
+
             if model_info.params.get("repeat_last_n", None):
                 payload["options"]["repeat_last_n"] = model_info.params.get(
                     "repeat_last_n", None

src/lib/components/chat/Settings/Advanced/AdvancedParams.svelte

@@ -21,6 +21,8 @@
 		top_p: null,
 		tfs_z: null,
 		num_ctx: null,
+		num_batch: null,
+		num_keep: null,
 		max_tokens: null,
 		use_mmap: null,
 		use_mlock: null,
@@ -565,6 +567,98 @@
 		{/if}
 	</div>
 
+	<div class=" py-0.5 w-full justify-between">
+		<div class="flex w-full justify-between">
+			<div class=" self-center text-xs font-medium">{$i18n.t('Batch Size (num_batch)')}</div>
+
+			<button
+				class="p-1 px-3 text-xs flex rounded transition"
+				type="button"
+				on:click={() => {
+					params.num_batch = (params?.num_batch ?? null) === null ? 512 : null;
+				}}
+			>
+				{#if (params?.num_batch ?? null) === null}
+					<span class="ml-2 self-center">{$i18n.t('Default')}</span>
+				{:else}
+					<span class="ml-2 self-center">{$i18n.t('Custom')}</span>
+				{/if}
+			</button>
+		</div>
+
+		{#if (params?.num_batch ?? null) !== null}
+			<div class="flex mt-0.5 space-x-2">
+				<div class=" flex-1">
+					<input
+						id="steps-range"
+						type="range"
+						min="256"
+						max="8192"
+						step="256"
+						bind:value={params.num_batch}
+						class="w-full h-2 rounded-lg appearance-none cursor-pointer dark:bg-gray-700"
+					/>
+				</div>
+				<div class="">
+					<input
+						bind:value={params.num_batch}
+						type="number"
+						class=" bg-transparent text-center w-14"
+						min="256"
+						step="256"
+					/>
+				</div>
+			</div>
+		{/if}
+	</div>
+
+	<div class=" py-0.5 w-full justify-between">
+		<div class="flex w-full justify-between">
+			<div class=" self-center text-xs font-medium">
+				{$i18n.t('Tokens To Keep On Context Refresh (num_keep)')}
+			</div>
+
+			<button
+				class="p-1 px-3 text-xs flex rounded transition"
+				type="button"
+				on:click={() => {
+					params.num_keep = (params?.num_keep ?? null) === null ? 24 : null;
+				}}
+			>
+				{#if (params?.num_keep ?? null) === null}
+					<span class="ml-2 self-center">{$i18n.t('Default')}</span>
+				{:else}
+					<span class="ml-2 self-center">{$i18n.t('Custom')}</span>
+				{/if}
+			</button>
+		</div>
+
+		{#if (params?.num_keep ?? null) !== null}
+			<div class="flex mt-0.5 space-x-2">
+				<div class=" flex-1">
+					<input
+						id="steps-range"
+						type="range"
+						min="-1"
+						max="10240000"
+						step="1"
+						bind:value={params.num_keep}
+						class="w-full h-2 rounded-lg appearance-none cursor-pointer dark:bg-gray-700"
+					/>
+				</div>
+				<div class="">
+					<input
+						bind:value={params.num_keep}
+						type="number"
+						class=" bg-transparent text-center w-14"
+						min="-1"
+						step="1"
+					/>
+				</div>
+			</div>
+		{/if}
+	</div>
+
 	<div class=" py-0.5 w-full justify-between">
 		<div class="flex w-full justify-between">
 			<div class=" self-center text-xs font-medium">{$i18n.t('Max Tokens (num_predict)')}</div>

src/lib/components/chat/Settings/General.svelte

@@ -55,6 +55,8 @@
 		stop: null,
 		tfs_z: null,
 		num_ctx: null,
+		num_batch: null,
+		num_keep: null,
 		max_tokens: null
 	};
 
@@ -308,6 +310,8 @@
 						top_p: params.top_p !== null ? params.top_p : undefined,
 						tfs_z: params.tfs_z !== null ? params.tfs_z : undefined,
 						num_ctx: params.num_ctx !== null ? params.num_ctx : undefined,
+						num_batch: params.num_batch !== null ? params.num_batch : undefined,
+						num_keep: params.num_keep !== null ? params.num_keep : undefined,
 						max_tokens: params.max_tokens !== null ? params.max_tokens : undefined,
 						use_mmap: params.use_mmap !== null ? params.use_mmap : undefined,
 						use_mlock: params.use_mlock !== null ? params.use_mlock : undefined,

src/lib/i18n/locales/ar-BH/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "استجابة خطاء",
 	"Banners": "لافتات",
 	"Base Model (From)": "النموذج الأساسي (من)",
+	"Batch Size (num_batch)": "",
 	"before": "قبل",
 	"Being lazy": "كون كسول",
 	"Brave Search API Key": "مفتاح واجهة برمجة تطبيقات البحث الشجاع",
@@ -523,6 +524,7 @@
 	"Today": "اليوم",
 	"Toggle settings": "فتح وأغلاق الاعدادات",
 	"Toggle sidebar": "فتح وأغلاق الشريط الجانبي",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/bg-BG/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "Невалиден отговор от API",
 	"Banners": "Банери",
 	"Base Model (From)": "Базов модел (от)",
+	"Batch Size (num_batch)": "",
 	"before": "преди",
 	"Being lazy": "Да бъдеш мързелив",
 	"Brave Search API Key": "Смел ключ за API за търсене",
@@ -519,6 +520,7 @@
 	"Today": "днес",
 	"Toggle settings": "Toggle settings",
 	"Toggle sidebar": "Toggle sidebar",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/bn-BD/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "খারাপ প্রতিক্রিয়া",
 	"Banners": "ব্যানার",
 	"Base Model (From)": "বেস মডেল (থেকে)",
+	"Batch Size (num_batch)": "",
 	"before": "পূর্ববর্তী",
 	"Being lazy": "অলস হওয়া",
 	"Brave Search API Key": "সাহসী অনুসন্ধান API কী",
@@ -519,6 +520,7 @@
 	"Today": "আজ",
 	"Toggle settings": "সেটিংস টোগল",
 	"Toggle sidebar": "সাইডবার টোগল",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/ca-ES/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "Resposta Erroni",
 	"Banners": "Banners",
 	"Base Model (From)": "Model base (des de)",
+	"Batch Size (num_batch)": "",
 	"before": "abans",
 	"Being lazy": "Ser l'estupidez",
 	"Brave Search API Key": "Clau API Brave Search",
@@ -520,6 +521,7 @@
 	"Today": "Avui",
 	"Toggle settings": "Commuta configuracions",
 	"Toggle sidebar": "Commuta barra lateral",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/ceb-PH/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "",
 	"Banners": "",
 	"Base Model (From)": "",
+	"Batch Size (num_batch)": "",
 	"before": "",
 	"Being lazy": "",
 	"Brave Search API Key": "",
@@ -519,6 +520,7 @@
 	"Today": "",
 	"Toggle settings": "I-toggle ang mga setting",
 	"Toggle sidebar": "I-toggle ang sidebar",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Ibabaw nga P",

src/lib/i18n/locales/de-DE/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "Schlechte Antwort",
 	"Banners": "Banner",
 	"Base Model (From)": "Basismodell (von)",
+	"Batch Size (num_batch)": "",
 	"before": "bereits geteilt",
 	"Being lazy": "Faul sein",
 	"Brave Search API Key": "API-Schlüssel für die Brave-Suche",
@@ -519,6 +520,7 @@
 	"Today": "Heute",
 	"Toggle settings": "Einstellungen umschalten",
 	"Toggle sidebar": "Seitenleiste umschalten",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/dg-DG/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "",
 	"Banners": "",
 	"Base Model (From)": "",
+	"Batch Size (num_batch)": "",
 	"before": "",
 	"Being lazy": "",
 	"Brave Search API Key": "",
@@ -519,6 +520,7 @@
 	"Today": "",
 	"Toggle settings": "Toggle settings much toggle",
 	"Toggle sidebar": "Toggle sidebar much toggle",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K very top",
 	"Top P": "Top P very top",

src/lib/i18n/locales/en-GB/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "",
 	"Banners": "",
 	"Base Model (From)": "",
+	"Batch Size (num_batch)": "",
 	"before": "",
 	"Being lazy": "",
 	"Brave Search API Key": "",
@@ -519,6 +520,7 @@
 	"Today": "",
 	"Toggle settings": "",
 	"Toggle sidebar": "",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "",
 	"Top P": "",

src/lib/i18n/locales/en-US/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "",
 	"Banners": "",
 	"Base Model (From)": "",
+	"Batch Size (num_batch)": "",
 	"before": "",
 	"Being lazy": "",
 	"Brave Search API Key": "",
@@ -519,6 +520,7 @@
 	"Today": "",
 	"Toggle settings": "",
 	"Toggle sidebar": "",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "",
 	"Top P": "",

src/lib/i18n/locales/es-ES/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "Respuesta incorrecta",
 	"Banners": "Banners",
 	"Base Model (From)": "Modelo base (desde)",
+	"Batch Size (num_batch)": "",
 	"before": "antes",
 	"Being lazy": "Ser perezoso",
 	"Brave Search API Key": "Clave de API de Brave Search",
@@ -520,6 +521,7 @@
 	"Today": "Hoy",
 	"Toggle settings": "Alternar configuración",
 	"Toggle sidebar": "Alternar barra lateral",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/fa-IR/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "پاسخ خوب نیست",
 	"Banners": "بنر",
 	"Base Model (From)": "مدل پایه (از)",
+	"Batch Size (num_batch)": "",
 	"before": "قبل",
 	"Being lazy": "حالت سازنده",
 	"Brave Search API Key": "کلید API جستجوی شجاع",
@@ -519,6 +520,7 @@
 	"Today": "امروز",
 	"Toggle settings": "نمایش/عدم نمایش تنظیمات",
 	"Toggle sidebar": "نمایش/عدم نمایش نوار کناری",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/fi-FI/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "Epäkelpo vastaus",
 	"Banners": "Bannerit",
 	"Base Model (From)": "Perusmalli (alkaen)",
+	"Batch Size (num_batch)": "",
 	"before": "ennen",
 	"Being lazy": "Oli laiska",
 	"Brave Search API Key": "Brave Search API -avain",
@@ -519,6 +520,7 @@
 	"Today": "Tänään",
 	"Toggle settings": "Kytke asetukset",
 	"Toggle sidebar": "Kytke sivupalkki",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/fr-CA/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "Mauvaise réponse",
 	"Banners": "Bannières",
 	"Base Model (From)": "Modèle de base (à partir de)",
+	"Batch Size (num_batch)": "",
 	"before": "avant",
 	"Being lazy": "En manque de temps",
 	"Brave Search API Key": "Clé d’API de recherche brave",
@@ -520,6 +521,7 @@
 	"Today": "Aujourd'hui",
 	"Toggle settings": "Basculer les paramètres",
 	"Toggle sidebar": "Basculer la barre latérale",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/fr-FR/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "Mauvaise Réponse",
 	"Banners": "Bannières",
 	"Base Model (From)": "Modèle de Base (De)",
+	"Batch Size (num_batch)": "",
 	"before": "avant",
 	"Being lazy": "Est paresseux",
 	"Brave Search API Key": "Clé API Brave Search",
@@ -520,6 +521,7 @@
 	"Today": "Aujourd'hui",
 	"Toggle settings": "Basculer les paramètres",
 	"Toggle sidebar": "Basculer la barre latérale",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/he-IL/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "תגובה שגויה",
 	"Banners": "באנרים",
 	"Base Model (From)": "דגם בסיס (מ)",
+	"Batch Size (num_batch)": "",
 	"before": "לפני",
 	"Being lazy": "להיות עצלן",
 	"Brave Search API Key": "מפתח API של חיפוש אמיץ",
@@ -520,6 +521,7 @@
 	"Today": "היום",
 	"Toggle settings": "החלפת מצב של הגדרות",
 	"Toggle sidebar": "החלפת מצב של סרגל הצד",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "Top K",
 	"Top P": "Top P",

src/lib/i18n/locales/hi-IN/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "ख़राब प्रतिक्रिया",
 	"Banners": "बैनर",
 	"Base Model (From)": "बेस मॉडल (से)",
+	"Batch Size (num_batch)": "",
 	"before": "पहले",
 	"Being lazy": "आलसी होना",
 	"Brave Search API Key": "Brave सर्च एपीआई कुंजी",
@@ -519,6 +520,7 @@
 	"Today": "आज",
 	"Toggle settings": "सेटिंग्स टॉगल करें",
 	"Toggle sidebar": "साइडबार टॉगल करें",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "",
 	"Top K": "शीर्ष  K",
 	"Top P": "शीर्ष  P",

src/lib/i18n/locales/hr-HR/translation.json

@@ -69,6 +69,7 @@
 	"Bad Response": "Loš odgovor",
 	"Banners": "Baneri",
 	"Base Model (From)": "Osnovni model (Od)",
+	"Batch Size (num_batch)": "",
 	"before": "prije",
 	"Being lazy": "Biti lijen",
 	"Brave Search API Key": "Brave tražilica - API ključ",
@@ -520,6 +521,7 @@
 	"Today": "Danas",
 	"Toggle settings": "Prebaci postavke",
 	"Toggle sidebar": "Prebaci bočnu traku",
+	"Tokens To Keep On Context Refresh (num_keep)": "",
 	"Tools": "Alati",
 	"Top K": "Top K",
 	"Top P": "Top P",