Merge pull request #2140 from cheahjs/feat/model-config

feat: configurable model name, description and vision capability

backend/apps/litellm/main.py

@@ -18,8 +18,9 @@ import requests
 from pydantic import BaseModel, ConfigDict
 from typing import Optional, List
 
+from apps.web.models.models import Models
 from utils.utils import get_verified_user, get_current_user, get_admin_user
-from config import SRC_LOG_LEVELS, ENV
+from config import SRC_LOG_LEVELS
 from constants import MESSAGES
 
 import os
@@ -77,7 +78,7 @@ with open(LITELLM_CONFIG_DIR, "r") as file:
 
 app.state.ENABLE_MODEL_FILTER = ENABLE_MODEL_FILTER.value
 app.state.MODEL_FILTER_LIST = MODEL_FILTER_LIST.value
-
+app.state.MODEL_CONFIG = Models.get_all_models()
 
 app.state.ENABLE = ENABLE_LITELLM
 app.state.CONFIG = litellm_config
@@ -241,6 +242,8 @@ async def get_models(user=Depends(get_current_user)):
                         )
                     )
 
+            for model in data["data"]:
+                add_custom_info_to_model(model)
             return data
         except Exception as e:
 
@@ -261,6 +264,14 @@ async def get_models(user=Depends(get_current_user)):
                         "object": "model",
                         "created": int(time.time()),
                         "owned_by": "openai",
+                        "custom_info": next(
+                            (
+                                item
+                                for item in app.state.MODEL_CONFIG
+                                if item.id == model["model_name"]
+                            ),
+                            None,
+                        ),
                     }
                     for model in app.state.CONFIG["model_list"]
                 ],
@@ -273,6 +284,12 @@ async def get_models(user=Depends(get_current_user)):
         }
 
 
+def add_custom_info_to_model(model: dict):
+    model["custom_info"] = next(
+        (item for item in app.state.MODEL_CONFIG if item.id == model["id"]), None
+    )
+
+
 @app.get("/model/info")
 async def get_model_list(user=Depends(get_admin_user)):
     return {"data": app.state.CONFIG["model_list"]}

backend/apps/ollama/main.py

@@ -29,7 +29,7 @@ import time
 from urllib.parse import urlparse
 from typing import Optional, List, Union
 
-
+from apps.web.models.models import Models
 from apps.web.models.users import Users
 from constants import ERROR_MESSAGES
 from utils.utils import (
@@ -67,6 +67,7 @@ app.state.config = AppConfig()
 
 app.state.config.ENABLE_MODEL_FILTER = ENABLE_MODEL_FILTER
 app.state.config.MODEL_FILTER_LIST = MODEL_FILTER_LIST
+app.state.MODEL_CONFIG = Models.get_all_models()
 
 
 app.state.config.ENABLE_OLLAMA_API = ENABLE_OLLAMA_API
@@ -191,12 +192,21 @@ async def get_all_models():
 
     else:
         models = {"models": []}
+        
+    for model in models["models"]:
+        add_custom_info_to_model(model)
 
     app.state.MODELS = {model["model"]: model for model in models["models"]}
 
     return models
 
 
+def add_custom_info_to_model(model: dict):
+    model["custom_info"] = next(
+        (item for item in app.state.MODEL_CONFIG if item.id == model["model"]), None
+    )
+
+
 @app.get("/api/tags")
 @app.get("/api/tags/{url_idx}")
 async def get_ollama_tags(

backend/apps/openai/main.py

@@ -10,7 +10,7 @@ import logging
 
 from pydantic import BaseModel
 
-
+from apps.web.models.models import Models
 from apps.web.models.users import Users
 from constants import ERROR_MESSAGES
 from utils.utils import (
@@ -52,6 +52,7 @@ app.state.config = AppConfig()
 
 app.state.config.ENABLE_MODEL_FILTER = ENABLE_MODEL_FILTER
 app.state.config.MODEL_FILTER_LIST = MODEL_FILTER_LIST
+app.state.MODEL_CONFIG = Models.get_all_models()
 
 
 app.state.config.ENABLE_OPENAI_API = ENABLE_OPENAI_API
@@ -249,10 +250,19 @@ async def get_all_models():
             )
         }
 
+        for model in models["data"]:
+            add_custom_info_to_model(model)
+
         log.info(f"models: {models}")
         app.state.MODELS = {model["id"]: model for model in models["data"]}
 
-        return models
+    return models
+
+
+def add_custom_info_to_model(model: dict):
+    model["custom_info"] = next(
+        (item for item in app.state.MODEL_CONFIG if item.id == model["id"]), None
+    )
 
 
 @app.get("/models")

backend/apps/web/internal/db.py

+import json
+
 from peewee import *
 from peewee_migrate import Router
 from playhouse.db_url import connect
@@ -8,6 +10,16 @@ import logging
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["DB"])
 
+
+class JSONField(TextField):
+    def db_value(self, value):
+        return json.dumps(value)
+
+    def python_value(self, value):
+        if value is not None:
+            return json.loads(value)
+
+
 # Check if the file exists
 if os.path.exists(f"{DATA_DIR}/ollama.db"):
     # Rename the file

backend/apps/web/internal/migrations/009_add_models.py

+"""Peewee migrations -- 009_add_models.py.
+
+Some examples (model - class or model name)::
+
+    > Model = migrator.orm['table_name']            # Return model in current state by name
+    > Model = migrator.ModelClass                   # Return model in current state by name
+
+    > migrator.sql(sql)                             # Run custom SQL
+    > migrator.run(func, *args, **kwargs)           # Run python function with the given args
+    > migrator.create_model(Model)                  # Create a model (could be used as decorator)
+    > migrator.remove_model(model, cascade=True)    # Remove a model
+    > migrator.add_fields(model, **fields)          # Add fields to a model
+    > migrator.change_fields(model, **fields)       # Change fields
+    > migrator.remove_fields(model, *field_names, cascade=True)
+    > migrator.rename_field(model, old_field_name, new_field_name)
+    > migrator.rename_table(model, new_table_name)
+    > migrator.add_index(model, *col_names, unique=False)
+    > migrator.add_not_null(model, *field_names)
+    > migrator.add_default(model, field_name, default)
+    > migrator.add_constraint(model, name, sql)
+    > migrator.drop_index(model, *col_names)
+    > migrator.drop_not_null(model, *field_names)
+    > migrator.drop_constraints(model, *constraints)
+
+"""
+
+from contextlib import suppress
+
+import peewee as pw
+from peewee_migrate import Migrator
+
+
+with suppress(ImportError):
+    import playhouse.postgres_ext as pw_pext
+
+
+def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
+    """Write your migrations here."""
+
+    @migrator.create_model
+    class Model(pw.Model):
+        id = pw.TextField(unique=True)
+        meta = pw.TextField()
+        base_model_id = pw.TextField(null=True)
+        name = pw.TextField()
+        params = pw.TextField()
+
+        class Meta:
+            table_name = "model"
+
+
+def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
+    """Write your rollback migrations here."""
+
+    migrator.remove_model("model")

backend/apps/web/models/models.py

+import json
+import logging
+from typing import Optional
+
+import peewee as pw
+from playhouse.shortcuts import model_to_dict
+from pydantic import BaseModel
+
+from apps.web.internal.db import DB, JSONField
+
+from config import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["MODELS"])
+
+
+####################
+# Models DB Schema
+####################
+
+
+# ModelParams is a model for the data stored in the params field of the Model table
+# It isn't currently used in the backend, but it's here as a reference
+class ModelParams(BaseModel):
+    pass
+
+
+# ModelMeta is a model for the data stored in the meta field of the Model table
+# It isn't currently used in the backend, but it's here as a reference
+class ModelMeta(BaseModel):
+    description: str
+    """
+        User-facing description of the model.
+    """
+
+    vision_capable: bool
+    """
+        A flag indicating if the model is capable of vision and thus image inputs
+    """
+
+
+class Model(pw.Model):
+    id = pw.TextField(unique=True)
+    """
+        The model's id as used in the API. If set to an existing model, it will override the model.
+    """
+
+    meta = JSONField()
+    """
+        Holds a JSON encoded blob of metadata, see `ModelMeta`.
+    """
+
+    base_model_id = pw.TextField(null=True)
+    """
+        An optional pointer to the actual model that should be used when proxying requests.
+        Currently unused - but will be used to support Modelfile like behaviour in the future
+    """
+
+    name = pw.TextField()
+    """
+        The human-readable display name of the model.
+    """
+
+    params = JSONField()
+    """
+        Holds a JSON encoded blob of parameters, see `ModelParams`.
+    """
+
+    class Meta:
+        database = DB
+
+
+class ModelModel(BaseModel):
+    id: str
+    meta: ModelMeta
+    base_model_id: Optional[str] = None
+    name: str
+    params: ModelParams
+
+
+####################
+# Forms
+####################
+
+
+class ModelsTable:
+
+    def __init__(
+        self,
+        db: pw.SqliteDatabase | pw.PostgresqlDatabase,
+    ):
+        self.db = db
+        self.db.create_tables([Model])
+
+    def get_all_models(self) -> list[ModelModel]:
+        return [ModelModel(**model_to_dict(model)) for model in Model.select()]
+
+    def update_all_models(self, models: list[ModelModel]) -> bool:
+        try:
+            with self.db.atomic():
+                # Fetch current models from the database
+                current_models = self.get_all_models()
+                current_model_dict = {model.id: model for model in current_models}
+
+                # Create a set of model IDs from the current models and the new models
+                current_model_keys = set(current_model_dict.keys())
+                new_model_keys = set(model.id for model in models)
+
+                # Determine which models need to be created, updated, or deleted
+                models_to_create = [
+                    model for model in models if model.id not in current_model_keys
+                ]
+                models_to_update = [
+                    model for model in models if model.id in current_model_keys
+                ]
+                models_to_delete = current_model_keys - new_model_keys
+
+                # Perform the necessary database operations
+                for model in models_to_create:
+                    Model.create(**model.model_dump())
+
+                for model in models_to_update:
+                    Model.update(**model.model_dump()).where(
+                        Model.id == model.id
+                    ).execute()
+
+                for model_id, model_source in models_to_delete:
+                    Model.delete().where(Model.id == model_id).execute()
+
+            return True
+        except Exception as e:
+            log.exception(e)
+            return False
+
+
+Models = ModelsTable(DB)

backend/main.py

@@ -36,9 +36,9 @@ from apps.web.main import app as webui_app
 
 import asyncio
 from pydantic import BaseModel
-from typing import List
-
+from typing import List, Optional
 
+from apps.web.models.models import Models, ModelModel
 from utils.utils import get_admin_user
 from apps.rag.utils import rag_messages
 
@@ -113,6 +113,8 @@ app.state.config = AppConfig()
 app.state.config.ENABLE_MODEL_FILTER = ENABLE_MODEL_FILTER
 app.state.config.MODEL_FILTER_LIST = MODEL_FILTER_LIST
 
+app.state.MODEL_CONFIG = Models.get_all_models()
+
 app.state.config.WEBHOOK_URL = WEBHOOK_URL
 
 origins = ["*"]
@@ -318,6 +320,33 @@ async def update_model_filter_config(
     }
 
 
+class SetModelConfigForm(BaseModel):
+    models: List[ModelModel]
+
+
+@app.post("/api/config/models")
+async def update_model_config(
+    form_data: SetModelConfigForm, user=Depends(get_admin_user)
+):
+    if not Models.update_all_models(form_data.models):
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=ERROR_MESSAGES.DEFAULT("Failed to update model config"),
+        )
+
+    ollama_app.state.MODEL_CONFIG = form_data.models
+    openai_app.state.MODEL_CONFIG = form_data.models
+    litellm_app.state.MODEL_CONFIG = form_data.models
+    app.state.MODEL_CONFIG = form_data.models
+
+    return {"models": app.state.MODEL_CONFIG}
+
+
+@app.get("/api/config/models")
+async def get_model_config(user=Depends(get_admin_user)):
+    return {"models": app.state.MODEL_CONFIG}
+
+
 @app.get("/api/webhook")
 async def get_webhook_url(user=Depends(get_admin_user)):
     return {

src/lib/apis/index.ts

@@ -196,3 +196,77 @@ export const updateWebhookUrl = async (token: string, url: string) => {
 
 	return res.url;
 };
+
+export const getModelConfig = async (token: string): Promise<GlobalModelConfig> => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_BASE_URL}/api/config/models`, {
+		method: 'GET',
+		headers: {
+			'Content-Type': 'application/json',
+			Authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			console.log(err);
+			error = err;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res.models;
+};
+
+export interface ModelConfig {
+	id: string;
+	name: string;
+	meta: ModelMeta;
+	base_model_id?: string;
+	params: ModelParams;
+}
+
+export interface ModelMeta {
+	description?: string;
+	vision_capable?: boolean;
+}
+
+export interface ModelParams {}
+
+export type GlobalModelConfig = ModelConfig[];
+
+export const updateModelConfig = async (token: string, config: GlobalModelConfig) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_BASE_URL}/api/config/models`, {
+		method: 'POST',
+		headers: {
+			'Content-Type': 'application/json',
+			Authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			models: config
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			console.log(err);
+			error = err;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};

src/lib/apis/litellm/index.ts

@@ -33,7 +33,8 @@ export const getLiteLLMModels = async (token: string = '') => {
 					id: model.id,
 					name: model.name ?? model.id,
 					external: true,
-					source: 'LiteLLM'
+					source: 'LiteLLM',
+					custom_info: model.custom_info
 				}))
 				.sort((a, b) => {
 					return a.name.localeCompare(b.name);

src/lib/apis/openai/index.ts

@@ -230,7 +230,12 @@ export const getOpenAIModels = async (token: string = '') => {
 
 	return models
 		? models
-				.map((model) => ({ id: model.id, name: model.name ?? model.id, external: true }))
+				.map((model) => ({
+					id: model.id,
+					name: model.name ?? model.id,
+					external: true,
+					custom_info: model.custom_info
+				}))
 				.sort((a, b) => {
 					return a.name.localeCompare(b.name);
 				})

src/lib/components/admin/Settings/Users.svelte

@@ -125,7 +125,7 @@
 												<option value="" disabled selected>{$i18n.t('Select a model')}</option>
 												{#each $models.filter((model) => model.id) as model}
 													<option value={model.id} class="bg-gray-100 dark:bg-gray-700"
-														>{model.name}</option
+														>{model.custom_info?.name ?? model.name}</option
 													>
 												{/each}
 											</select>

src/lib/components/chat/Chat.svelte

@@ -10,6 +10,7 @@
 		chatId,
 		chats,
 		config,
+		type Model,
 		modelfiles,
 		models,
 		settings,
@@ -60,7 +61,7 @@
 	let showModelSelector = true;
 
 	let selectedModels = [''];
-	let atSelectedModel = '';
+	let atSelectedModel: Model | undefined;
 
 	let selectedModelfile = null;
 	$: selectedModelfile =
@@ -328,75 +329,91 @@
 		const _chatId = JSON.parse(JSON.stringify($chatId));
 
 		await Promise.all(
-			(modelId ? [modelId] : atSelectedModel !== '' ? [atSelectedModel.id] : selectedModels).map(
-				async (modelId) => {
-					console.log('modelId', modelId);
-					const model = $models.filter((m) => m.id === modelId).at(0);
-
-					if (model) {
-						// Create response message
-						let responseMessageId = uuidv4();
-						let responseMessage = {
-							parentId: parentId,
-							id: responseMessageId,
-							childrenIds: [],
-							role: 'assistant',
-							content: '',
-							model: model.id,
-							userContext: null,
-							timestamp: Math.floor(Date.now() / 1000) // Unix epoch
-						};
-
-						// Add message to history and Set currentId to messageId
-						history.messages[responseMessageId] = responseMessage;
-						history.currentId = responseMessageId;
-
-						// Append messageId to childrenIds of parent message
-						if (parentId !== null) {
-							history.messages[parentId].childrenIds = [
-								...history.messages[parentId].childrenIds,
-								responseMessageId
-							];
-						}
-
-						await tick();
+			(modelId
+				? [modelId]
+				: atSelectedModel !== undefined
+				? [atSelectedModel.id]
+				: selectedModels
+			).map(async (modelId) => {
+				console.log('modelId', modelId);
+				const model = $models.filter((m) => m.id === modelId).at(0);
+
+				if (model) {
+					// If there are image files, check if model is vision capable
+					const hasImages = messages.some((message) =>
+						message.files?.some((file) => file.type === 'image')
+					);
+					if (hasImages && !(model.custom_info?.meta.vision_capable ?? true)) {
+						toast.error(
+							$i18n.t('Model {{modelName}} is not vision capable', {
+								modelName: model.custom_info?.name ?? model.name ?? model.id
+							})
+						);
+					}
 
-						let userContext = null;
-						if ($settings?.memory ?? false) {
-							if (userContext === null) {
-								const res = await queryMemory(localStorage.token, prompt).catch((error) => {
-									toast.error(error);
-									return null;
-								});
-
-								if (res) {
-									if (res.documents[0].length > 0) {
-										userContext = res.documents.reduce((acc, doc, index) => {
-											const createdAtTimestamp = res.metadatas[index][0].created_at;
-											const createdAtDate = new Date(createdAtTimestamp * 1000)
-												.toISOString()
-												.split('T')[0];
-											acc.push(`${index + 1}. [${createdAtDate}]. ${doc[0]}`);
-											return acc;
-										}, []);
-									}
+					// Create response message
+					let responseMessageId = uuidv4();
+					let responseMessage = {
+						parentId: parentId,
+						id: responseMessageId,
+						childrenIds: [],
+						role: 'assistant',
+						content: '',
+						model: model.id,
+						modelName: model.custom_info?.name ?? model.name ?? model.id,
+						userContext: null,
+						timestamp: Math.floor(Date.now() / 1000) // Unix epoch
+					};
+
+					// Add message to history and Set currentId to messageId
+					history.messages[responseMessageId] = responseMessage;
+					history.currentId = responseMessageId;
+
+					// Append messageId to childrenIds of parent message
+					if (parentId !== null) {
+						history.messages[parentId].childrenIds = [
+							...history.messages[parentId].childrenIds,
+							responseMessageId
+						];
+					}
 
-									console.log(userContext);
+					await tick();
+
+					let userContext = null;
+					if ($settings?.memory ?? false) {
+						if (userContext === null) {
+							const res = await queryMemory(localStorage.token, prompt).catch((error) => {
+								toast.error(error);
+								return null;
+							});
+
+							if (res) {
+								if (res.documents[0].length > 0) {
+									userContext = res.documents.reduce((acc, doc, index) => {
+										const createdAtTimestamp = res.metadatas[index][0].created_at;
+										const createdAtDate = new Date(createdAtTimestamp * 1000)
+											.toISOString()
+											.split('T')[0];
+										acc.push(`${index + 1}. [${createdAtDate}]. ${doc[0]}`);
+										return acc;
+									}, []);
 								}
+
+								console.log(userContext);
 							}
 						}
-						responseMessage.userContext = userContext;
+					}
+					responseMessage.userContext = userContext;
 
-						if (model?.external) {
-							await sendPromptOpenAI(model, prompt, responseMessageId, _chatId);
-						} else if (model) {
-							await sendPromptOllama(model, prompt, responseMessageId, _chatId);
-						}
-					} else {
-						toast.error($i18n.t(`Model {{modelId}} not found`, { modelId }));
+					if (model?.external) {
+						await sendPromptOpenAI(model, prompt, responseMessageId, _chatId);
+					} else if (model) {
+						await sendPromptOllama(model, prompt, responseMessageId, _chatId);
 					}
+				} else {
+					toast.error($i18n.t(`Model {{modelId}} not found`, { modelId }));
 				}
-			)
+			})
 		);
 
 		await chats.set(await getChatList(localStorage.token));
@@ -855,7 +872,7 @@
 		responseMessage.error = true;
 		responseMessage.content =
 			$i18n.t(`Uh-oh! There was an issue connecting to {{provider}}.`, {
-				provider: model.name ?? model.id
+				provider: model.custom_info?.name ?? model.name ?? model.id
 			}) +
 			'\n' +
 			errorMessage;
@@ -1049,6 +1066,7 @@
 		bind:prompt
 		bind:autoScroll
 		bind:selectedModel={atSelectedModel}
+		{selectedModels}
 		{messages}
 		{submitPrompt}
 		{stopResponse}

src/lib/components/chat/MessageInput.svelte

 <script lang="ts">
 	import { toast } from 'svelte-sonner';
 	import { onMount, tick, getContext } from 'svelte';
-	import { mobile, modelfiles, settings, showSidebar } from '$lib/stores';
+	import { type Model, mobile, modelfiles, settings, showSidebar, models } from '$lib/stores';
 	import { blobToFile, calculateSHA256, findWordIndices } from '$lib/utils';
 
 	import {
@@ -27,7 +27,8 @@
 	export let stopResponse: Function;
 
 	export let autoScroll = true;
-	export let selectedModel = '';
+	export let selectedAtModel: Model | undefined;
+	export let selectedModels: [''];
 
 	let chatTextAreaElement: HTMLTextAreaElement;
 	let filesInputElement;
@@ -52,6 +53,8 @@
 
 	let speechRecognition;
 
+	let visionCapableState = 'all';
+
 	$: if (prompt) {
 		if (chatTextAreaElement) {
 			chatTextAreaElement.style.height = '';
@@ -59,6 +62,20 @@
 		}
 	}
 
+	$: {
+		if (selectedAtModel || selectedModels) {
+			visionCapableState = checkModelsAreVisionCapable();
+			if (visionCapableState === 'none') {
+				// Remove all image files
+				const fileCount = files.length;
+				files = files.filter((file) => file.type != 'image');
+				if (files.length < fileCount) {
+					toast.warning($i18n.t('All selected models do not support image input, removed images'));
+				}
+			}
+		}
+	}
+
 	let mediaRecorder;
 	let audioChunks = [];
 	let isRecording = false;
@@ -326,6 +343,35 @@
 		}
 	};
 
+	const checkModelsAreVisionCapable = () => {
+		let modelsToCheck = [];
+		if (selectedAtModel !== undefined) {
+			modelsToCheck = [selectedAtModel.id];
+		} else {
+			modelsToCheck = selectedModels;
+		}
+		if (modelsToCheck.length == 0 || modelsToCheck[0] == '') {
+			return 'all';
+		}
+		let visionCapableCount = 0;
+		for (const modelName of modelsToCheck) {
+			const model = $models.find((m) => m.id === modelName);
+			if (!model) {
+				continue;
+			}
+			if (model.custom_info?.meta.vision_capable ?? true) {
+				visionCapableCount++;
+			}
+		}
+		if (visionCapableCount == modelsToCheck.length) {
+			return 'all';
+		} else if (visionCapableCount == 0) {
+			return 'none';
+		} else {
+			return 'some';
+		}
+	};
+
 	onMount(() => {
 		window.setTimeout(() => chatTextAreaElement?.focus(), 0);
 
@@ -358,6 +404,10 @@
 					inputFiles.forEach((file) => {
 						console.log(file, file.name.split('.').at(-1));
 						if (['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(file['type'])) {
+							if (visionCapableState == 'none') {
+								toast.error($i18n.t('Selected models do not support image inputs'));
+								return;
+							}
 							let reader = new FileReader();
 							reader.onload = (event) => {
 								files = [
@@ -494,12 +544,12 @@
 						bind:chatInputPlaceholder
 						{messages}
 						on:select={(e) => {
-							selectedModel = e.detail;
+							selectedAtModel = e.detail;
 							chatTextAreaElement?.focus();
 						}}
 					/>
 
-					{#if selectedModel !== ''}
+					{#if selectedAtModel !== undefined}
 						<div
 							class="px-3 py-2.5 text-left w-full flex justify-between items-center absolute bottom-0 left-0 right-0 bg-gradient-to-t from-50% from-white dark:from-gray-900"
 						>
@@ -508,21 +558,23 @@
 									crossorigin="anonymous"
 									alt="model profile"
 									class="size-5 max-w-[28px] object-cover rounded-full"
-									src={$modelfiles.find((modelfile) => modelfile.tagName === selectedModel.id)
+									src={$modelfiles.find((modelfile) => modelfile.tagName === selectedAtModel.id)
 										?.imageUrl ??
 										($i18n.language === 'dg-DG'
 											? `/doge.png`
 											: `${WEBUI_BASE_URL}/static/favicon.png`)}
 								/>
 								<div>
-									Talking to <span class=" font-medium">{selectedModel.name} </span>
+									Talking to <span class=" font-medium"
+										>{selectedAtModel.custom_info?.name ?? selectedAtModel.name}
+									</span>
 								</div>
 							</div>
 							<div>
 								<button
 									class="flex items-center"
 									on:click={() => {
-										selectedModel = '';
+										selectedAtModel = undefined;
 									}}
 								>
 									<XMark />
@@ -550,6 +602,12 @@
 									if (
 										['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(file['type'])
 									) {
+										if (visionCapableState === 'none') {
+											toast.error($i18n.t('Selected models do not support image inputs'));
+											inputFiles = null;
+											filesInputElement.value = '';
+											return;
+										}
 										let reader = new FileReader();
 										reader.onload = (event) => {
 											files = [
@@ -597,7 +655,32 @@
 								{#each files as file, fileIdx}
 									<div class=" relative group">
 										{#if file.type === 'image'}
-											<img src={file.url} alt="input" class=" h-16 w-16 rounded-xl object-cover" />
+											<div class="relative">
+												<img
+													src={file.url}
+													alt="input"
+													class=" h-16 w-16 rounded-xl object-cover"
+												/>
+												{#if visionCapableState === 'some'}
+													<Tooltip
+														className=" absolute top-0 left-0"
+														content={$i18n.t('A selected model does not support image input')}
+													>
+														<svg
+															xmlns="http://www.w3.org/2000/svg"
+															viewBox="0 0 24 24"
+															fill="currentColor"
+															class="w-6 h-6 fill-yellow-300"
+														>
+															<path
+																fill-rule="evenodd"
+																d="M9.401 3.003c1.155-2 4.043-2 5.197 0l7.355 12.748c1.154 2-.29 4.5-2.599 4.5H4.645c-2.309 0-3.752-2.5-2.598-4.5L9.4 3.003ZM12 8.25a.75.75 0 0 1 .75.75v3.75a.75.75 0 0 1-1.5 0V9a.75.75 0 0 1 .75-.75Zm0 8.25a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Z"
+																clip-rule="evenodd"
+															/>
+														</svg>
+													</Tooltip>
+												{/if}
+											</div>
 										{:else if file.type === 'doc'}
 											<div
 												class="h-16 w-[15rem] flex items-center space-x-3 px-2.5 dark:bg-gray-600 rounded-xl border border-gray-200 dark:border-none"
@@ -883,7 +966,7 @@
 
 									if (e.key === 'Escape') {
 										console.log('Escape');
-										selectedModel = '';
+										selectedAtModel = undefined;
 									}
 								}}
 								rows="1"

src/lib/components/chat/MessageInput/Models.svelte

@@ -21,8 +21,10 @@
 	let filteredModels = [];
 
 	$: filteredModels = $models
-		.filter((p) => p.name.includes(prompt.split(' ')?.at(0)?.substring(1) ?? ''))
-		.sort((a, b) => a.name.localeCompare(b.name));
+		.filter((p) =>
+			(p.custom_info?.name ?? p.name).includes(prompt.split(' ')?.at(0)?.substring(1) ?? '')
+		)
+		.sort((a, b) => (a.custom_info?.name ?? a.name).localeCompare(b.custom_info?.name ?? b.name));
 
 	$: if (prompt) {
 		selectedIdx = 0;
@@ -156,7 +158,7 @@
 								on:focus={() => {}}
 							>
 								<div class=" font-medium text-black line-clamp-1">
-									{model.name}
+									{model.custom_info?.name ?? model.name}
 								</div>
 
 								<!-- <div class=" text-xs text-gray-600 line-clamp-1">

src/lib/components/chat/Messages/ResponseMessage.svelte

@@ -347,7 +347,7 @@
 				{#if message.model in modelfiles}
 					{modelfiles[message.model]?.title}
 				{:else}
-					{message.model ? ` ${message.model}` : ''}
+					{message.modelName ? ` ${message.modelName}` : message.model ? ` ${message.model}` : ''}
 				{/if}
 
 				{#if message.timestamp}

src/lib/components/chat/ModelSelector.svelte

@@ -49,7 +49,7 @@
 							.filter((model) => model.name !== 'hr')
 							.map((model) => ({
 								value: model.id,
-								label: model.name,
+								label: model.custom_info?.name ?? model.name,
 								info: model
 							}))}
 						bind:value={selectedModel}

src/lib/components/chat/ModelSelector/Selector.svelte

@@ -12,7 +12,12 @@
 
 	import { user, MODEL_DOWNLOAD_POOL, models, mobile } from '$lib/stores';
 	import { toast } from 'svelte-sonner';
-	import { capitalizeFirstLetter, getModels, splitStream } from '$lib/utils';
+	import {
+		capitalizeFirstLetter,
+		getModels,
+		sanitizeResponseContent,
+		splitStream
+	} from '$lib/utils';
 	import Tooltip from '$lib/components/common/Tooltip.svelte';
 
 	const i18n = getContext('i18n');
@@ -23,7 +28,12 @@
 	export let searchEnabled = true;
 	export let searchPlaceholder = $i18n.t('Search a model');
 
-	export let items = [{ value: 'mango', label: 'Mango' }];
+	export let items: {
+		label: string;
+		value: string;
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		[key: string]: any;
+	} = [];
 
 	export let className = 'w-[30rem]';
 
@@ -250,8 +260,8 @@
 							<!-- {JSON.stringify(item.info)} -->
 
 							{#if item.info.external}
-								<Tooltip content={item.info?.source ?? 'External'}>
-									<div class=" mr-2">
+								<Tooltip content={`${item.info?.source ?? 'External'}`}>
+									<div class="">
 										<svg
 											xmlns="http://www.w3.org/2000/svg"
 											viewBox="0 0 16 16"
@@ -279,7 +289,7 @@
 											: ''
 									}${item.info.size ? `(${(item.info.size / 1024 ** 3).toFixed(1)}GB)` : ''}`}
 								>
-									<div class=" mr-2">
+									<div class="">
 										<svg
 											xmlns="http://www.w3.org/2000/svg"
 											fill="none"
@@ -297,8 +307,31 @@
 									</div>
 								</Tooltip>
 							{/if}
+							{#if item.info?.custom_info?.meta.description}
+								<Tooltip
+									content={`${sanitizeResponseContent(
+										item.info.custom_info?.meta.description
+									).replaceAll('\n', '<br>')}`}
+								>
+									<div class="">
+										<svg
+											xmlns="http://www.w3.org/2000/svg"
+											fill="none"
+											viewBox="0 0 24 24"
+											stroke-width="1.5"
+											stroke="currentColor"
+											class="w-4 h-4"
+										>
+											<path
+												stroke-linecap="round"
+												stroke-linejoin="round"
+												d="M9.879 7.519c1.171-1.025 3.071-1.025 4.242 0 1.172 1.025 1.172 2.687 0 3.712-.203.179-.43.326-.67.442-.745.361-1.45.999-1.45 1.827v.75M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Zm-9 5.25h.008v.008H12v-.008Z"
+											/>
+										</svg>
+									</div>
+								</Tooltip>
+							{/if}
 						</div>
-
 						{#if value === item.value}
 							<div class="ml-auto">
 								<Check />

src/lib/components/chat/Settings/Interface.svelte

@@ -298,7 +298,10 @@
 						{#each $models as model}
 							{#if model.size != null}
 								<option value={model.name} class="bg-gray-100 dark:bg-gray-700">
-									{model.name + ' (' + (model.size / 1024 ** 3).toFixed(1) + ' GB)'}
+									{(model.custom_info?.name ?? model.name) +
+										' (' +
+										(model.size / 1024 ** 3).toFixed(1) +
+										' GB)'}
 								</option>
 							{/if}
 						{/each}
@@ -316,7 +319,7 @@
 						{#each $models as model}
 							{#if model.name !== 'hr'}
 								<option value={model.name} class="bg-gray-100 dark:bg-gray-700">
-									{model.name}
+									{model.custom_info?.name ?? model.name}
 								</option>
 							{/if}
 						{/each}

src/lib/components/chat/Settings/Models.svelte

@@ -13,10 +13,11 @@
 		uploadModel
 	} from '$lib/apis/ollama';
 	import { WEBUI_API_BASE_URL, WEBUI_BASE_URL } from '$lib/constants';
-	import { WEBUI_NAME, models, MODEL_DOWNLOAD_POOL, user } from '$lib/stores';
+	import { WEBUI_NAME, models, MODEL_DOWNLOAD_POOL, user, config } from '$lib/stores';
 	import { splitStream } from '$lib/utils';
 	import { onMount, getContext } from 'svelte';
 	import { addLiteLLMModel, deleteLiteLLMModel, getLiteLLMModelInfo } from '$lib/apis/litellm';
+	import { getModelConfig, type GlobalModelConfig, updateModelConfig } from '$lib/apis';
 	import Tooltip from '$lib/components/common/Tooltip.svelte';
 
 	const i18n = getContext('i18n');
@@ -67,6 +68,23 @@
 
 	let deleteModelTag = '';
 
+	// Model configuration
+	let modelConfig: GlobalModelConfig;
+	let showModelInfo = false;
+	let selectedModelId = '';
+	let modelName = '';
+	let modelDescription = '';
+	let modelIsVisionCapable = false;
+
+	const onModelInfoIdChange = () => {
+		const model = $models.find((m) => m.id === selectedModelId);
+		if (model) {
+			modelName = model.custom_info?.name ?? model.name;
+			modelDescription = model.custom_info?.meta.description ?? '';
+			modelIsVisionCapable = model.custom_info?.meta.vision_capable ?? false;
+		}
+	};
+
 	const updateModelsHandler = async () => {
 		for (const model of $models.filter(
 			(m) =>
@@ -492,18 +510,79 @@
 		models.set(await getModels());
 	};
 
-	onMount(async () => {
-		OLLAMA_URLS = await getOllamaUrls(localStorage.token).catch((error) => {
-			toast.error(error);
-			return [];
+	const addModelInfoHandler = async () => {
+		if (!selectedModelId) {
+			return;
+		}
+		let model = $models.find((m) => m.id === selectedModelId);
+		if (!model) {
+			return;
+		}
+		// Remove any existing config
+		modelConfig = modelConfig.filter(
+			(m) => !(m.id === selectedModelId)
+		);
+		// Add new config
+		modelConfig.push({
+			id: selectedModelId,
+			name: modelName,
+			params: {},
+			meta: {
+				description: modelDescription,
+				vision_capable: modelIsVisionCapable
+			}
 		});
+		await updateModelConfig(localStorage.token, modelConfig);
+		toast.success(
+			$i18n.t('Model info for {{modelName}} added successfully', { modelName: selectedModelId })
+		);
+		models.set(await getModels());
+	};
 
-		if (OLLAMA_URLS.length > 0) {
-			selectedOllamaUrlIdx = 0;
+	const deleteModelInfoHandler = async () => {
+		if (!selectedModelId) {
+			return;
+		}
+		let model = $models.find((m) => m.id === selectedModelId);
+		if (!model) {
+			return;
 		}
+		modelConfig = modelConfig.filter(
+			(m) => !(m.id === selectedModelId)
+		);
+		await updateModelConfig(localStorage.token, modelConfig);
+		toast.success(
+			$i18n.t('Model info for {{modelName}} deleted successfully', { modelName: selectedModelId })
+		);
+		models.set(await getModels());
+	};
 
-		ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => false);
-		liteLLMModelInfo = await getLiteLLMModelInfo(localStorage.token);
+	const toggleIsVisionCapable = () => {
+		modelIsVisionCapable = !modelIsVisionCapable;
+	};
+
+	onMount(async () => {
+		await Promise.all([
+			(async () => {
+				OLLAMA_URLS = await getOllamaUrls(localStorage.token).catch((error) => {
+					toast.error(error);
+					return [];
+				});
+
+				if (OLLAMA_URLS.length > 0) {
+					selectedOllamaUrlIdx = 0;
+				}
+			})(),
+			(async () => {
+				liteLLMModelInfo = await getLiteLLMModelInfo(localStorage.token);
+			})(),
+			(async () => {
+				modelConfig = await getModelConfig(localStorage.token);
+			})(),
+			(async () => {
+				ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => false);
+			})()
+		]);
 	});
 </script>
 
@@ -587,24 +666,28 @@
 											viewBox="0 0 24 24"
 											fill="currentColor"
 											xmlns="http://www.w3.org/2000/svg"
-											><style>
+										>
+											<style>
 												.spinner_ajPY {
 													transform-origin: center;
 													animation: spinner_AtaB 0.75s infinite linear;
 												}
+
 												@keyframes spinner_AtaB {
 													100% {
 														transform: rotate(360deg);
 													}
 												}
-											</style><path
+											</style>
+											<path
 												d="M12,1A11,11,0,1,0,23,12,11,11,0,0,0,12,1Zm0,19a8,8,0,1,1,8-8A8,8,0,0,1,12,20Z"
 												opacity=".25"
-											/><path
+											/>
+											<path
 												d="M10.14,1.16a11,11,0,0,0-9,8.92A1.59,1.59,0,0,0,2.46,12,1.52,1.52,0,0,0,4.11,10.7a8,8,0,0,1,6.66-6.61A1.42,1.42,0,0,0,12,2.69h0A1.57,1.57,0,0,0,10.14,1.16Z"
 												class="spinner_ajPY"
-											/></svg
-										>
+											/>
+										</svg>
 									</div>
 								{:else}
 									<svg
@@ -705,7 +788,10 @@
 									{/if}
 									{#each $models.filter((m) => m.size != null && (selectedOllamaUrlIdx === null ? true : (m?.urls ?? []).includes(selectedOllamaUrlIdx))) as model}
 										<option value={model.name} class="bg-gray-100 dark:bg-gray-700"
-											>{model.name + ' (' + (model.size / 1024 ** 3).toFixed(1) + ' GB)'}</option
+											>{(model.custom_info?.name ?? model.name) +
+												' (' +
+												(model.size / 1024 ** 3).toFixed(1) +
+												' GB)'}</option
 										>
 									{/each}
 								</select>
@@ -833,24 +919,28 @@
 													viewBox="0 0 24 24"
 													fill="currentColor"
 													xmlns="http://www.w3.org/2000/svg"
-													><style>
+												>
+													<style>
 														.spinner_ajPY {
 															transform-origin: center;
 															animation: spinner_AtaB 0.75s infinite linear;
 														}
+
 														@keyframes spinner_AtaB {
 															100% {
 																transform: rotate(360deg);
 															}
 														}
-													</style><path
+													</style>
+													<path
 														d="M12,1A11,11,0,1,0,23,12,11,11,0,0,0,12,1Zm0,19a8,8,0,1,1,8-8A8,8,0,0,1,12,20Z"
 														opacity=".25"
-													/><path
+													/>
+													<path
 														d="M10.14,1.16a11,11,0,0,0-9,8.92A1.59,1.59,0,0,0,2.46,12,1.52,1.52,0,0,0,4.11,10.7a8,8,0,0,1,6.66-6.61A1.42,1.42,0,0,0,12,2.69h0A1.57,1.57,0,0,0,10.14,1.16Z"
 														class="spinner_ajPY"
-													/></svg
-												>
+													/>
+												</svg>
 											</div>
 										{:else}
 											<svg
@@ -932,6 +1022,7 @@
 			<hr class=" dark:border-gray-700 my-2" />
 		{/if}
 
+		<!--TODO: Hide LiteLLM options when ENABLE_LITELLM=false-->
 		<div class=" space-y-3">
 			<div class="mt-2 space-y-3 pr-1.5">
 				<div>
@@ -1126,6 +1217,146 @@
 					{/if}
 				</div>
 			</div>
+			<hr class=" dark:border-gray-700 my-2" />
+		</div>
+
+		<div class=" space-y-3">
+			<div class="mt-2 space-y-3 pr-1.5">
+				<div>
+					<div class="mb-2">
+						<div class="flex justify-between items-center text-xs">
+							<div class=" text-sm font-medium">{$i18n.t('Manage Model Information')}</div>
+							<button
+								class=" text-xs font-medium text-gray-500"
+								type="button"
+								on:click={() => {
+									showModelInfo = !showModelInfo;
+								}}>{showModelInfo ? $i18n.t('Hide') : $i18n.t('Show')}</button
+							>
+						</div>
+					</div>
+
+					{#if showModelInfo}
+						<div>
+							<div class="flex justify-between items-center text-xs">
+								<div class=" text-sm font-medium">{$i18n.t('Current Models')}</div>
+							</div>
+
+							<div class="flex gap-2">
+								<div class="flex-1 pb-1">
+									<select
+										class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
+										bind:value={selectedModelId}
+										on:change={onModelInfoIdChange}
+									>
+										{#if !selectedModelId}
+											<option value="" disabled selected>{$i18n.t('Select a model')}</option>
+										{/if}
+										{#each $models as model}
+											<option value={model.id} class="bg-gray-100 dark:bg-gray-700"
+												>{'details' in model
+													? 'Ollama'
+													: model.source === 'LiteLLM'
+													? 'LiteLLM'
+													: 'OpenAI'}: {model.name}{`${
+													model.custom_info?.name ? ' - ' + model.custom_info?.name : ''
+												}`}</option
+											>
+										{/each}
+									</select>
+								</div>
+								<button
+									class="px-2.5 bg-gray-100 hover:bg-gray-200 text-gray-800 dark:bg-gray-850 dark:hover:bg-gray-800 dark:text-gray-100 rounded-lg transition"
+									on:click={() => {
+										deleteModelInfoHandler();
+									}}
+								>
+									<svg
+										xmlns="http://www.w3.org/2000/svg"
+										viewBox="0 0 16 16"
+										fill="currentColor"
+										class="w-4 h-4"
+									>
+										<path
+											fill-rule="evenodd"
+											d="M5 3.25V4H2.75a.75.75 0 0 0 0 1.5h.3l.815 8.15A1.5 1.5 0 0 0 5.357 15h5.285a1.5 1.5 0 0 0 1.493-1.35l.815-8.15h.3a.75.75 0 0 0 0-1.5H11v-.75A2.25 2.25 0 0 0 8.75 1h-1.5A2.25 2.25 0 0 0 5 3.25Zm2.25-.75a.75.75 0 0 0-.75.75V4h3v-.75a.75.75 0 0 0-.75-.75h-1.5ZM6.05 6a.75.75 0 0 1 .787.713l.275 5.5a.75.75 0 0 1-1.498.075l-.275-5.5A.75.75 0 0 1 6.05 6Zm3.9 0a.75.75 0 0 1 .712.787l-.275 5.5a.75.75 0 0 1-1.498-.075l.275-5.5a.75.75 0 0 1 .786-.711Z"
+											clip-rule="evenodd"
+										/>
+									</svg>
+								</button>
+							</div>
+
+							{#if selectedModelId}
+								<div>
+									<div class=" mb-1.5 text-sm font-medium">{$i18n.t('Model Display Name')}</div>
+									<div class="flex w-full mb-1.5">
+										<div class="flex-1 mr-2">
+											<input
+												class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
+												placeholder={$i18n.t('Enter Model Display Name')}
+												bind:value={modelName}
+												autocomplete="off"
+											/>
+										</div>
+
+										<button
+											class="px-2.5 bg-gray-100 hover:bg-gray-200 text-gray-800 dark:bg-gray-850 dark:hover:bg-gray-800 dark:text-gray-100 rounded-lg transition"
+											on:click={() => {
+												addModelInfoHandler();
+											}}
+										>
+											<svg
+												xmlns="http://www.w3.org/2000/svg"
+												viewBox="0 0 16 16"
+												fill="currentColor"
+												class="w-4 h-4"
+											>
+												<path
+													d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
+												/>
+											</svg>
+										</button>
+									</div>
+								</div>
+
+								<div>
+									<div class=" mb-1.5 text-sm font-medium">{$i18n.t('Model Description')}</div>
+
+									<div class="flex w-full">
+										<div class="flex-1">
+											<textarea
+												class="px-3 py-1.5 text-sm w-full bg-transparent border dark:border-gray-600 outline-none rounded-lg -mb-1"
+												rows="2"
+												bind:value={modelDescription}
+											/>
+										</div>
+									</div>
+								</div>
+
+								<div class="py-0.5 flex w-full justify-between">
+									<div class=" self-center text-sm font-medium">
+										{$i18n.t('Is Model Vision Capable')}
+									</div>
+
+									<button
+										class="p-1 px-3sm flex rounded transition"
+										on:click={() => {
+											toggleIsVisionCapable();
+										}}
+										type="button"
+									>
+										{#if modelIsVisionCapable === true}
+											<span class="ml-2 self-center">{$i18n.t('Yes')}</span>
+										{:else}
+											<span class="ml-2 self-center">{$i18n.t('No')}</span>
+										{/if}
+									</button>
+								</div>
+							{/if}
+						</div>
+					{/if}
+				</div>
+			</div>
 		</div>
 	</div>
 </div>

src/lib/components/common/Tooltip.svelte

@@ -5,6 +5,7 @@
 	export let placement = 'top';
 	export let content = `I'm a tooltip!`;
 	export let touch = true;
+	export let className = 'flex';
 
 	let tooltipElement;
 	let tooltipInstance;
@@ -29,6 +30,6 @@
 	});
 </script>
 
-<div bind:this={tooltipElement} aria-label={content} class="flex">
+<div bind:this={tooltipElement} aria-label={content} class={className}>
 	<slot />
 </div>