Merge pull request #1187 from open-webui/dev

0.1.113

CHANGELOG.md

@@ -5,6 +5,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.113] - 2024-03-18
+
+### Added
+
+- 🌍 **Localization**: You can now change the UI language in Settings > General. We support Ukrainian, German, Farsi (Persian), Traditional and Simplified Chinese and French translations. You can help us to translate the UI into your language! More info in our [CONTRIBUTION.md](https://github.com/open-webui/open-webui/blob/main/docs/CONTRIBUTING.md#-translations-and-internationalization).
+- 🎨 **System-wide Theme**: Introducing a new system-wide theme for enhanced visual experience.
+
+### Fixed
+
+- 🌑 **Dark Background on Select Fields**: Improved readability by adding a dark background to select fields, addressing issues on certain browsers/devices.
+- **Multiple OPENAI_API_BASE_URLS Issue**: Resolved issue where multiple base URLs caused conflicts when one wasn't functioning.
+- **RAG Encoding Issue**: Fixed encoding problem in RAG.
+- **npm Audit Fix**: Addressed npm audit findings.
+- **Reduced Scroll Threshold**: Improved auto-scroll experience by reducing the scroll threshold from 50px to 5px.
+
+### Changed
+
+- 🔄 **Sidebar UI Update**: Updated sidebar UI to feature a chat menu dropdown, replacing two icons for improved navigation.
+
 ## [0.1.112] - 2024-03-15
 
 ### Fixed

README.md

@@ -11,7 +11,7 @@
 [![Discord](https://img.shields.io/badge/Discord-Open_WebUI-blue?logo=discord&logoColor=white)](https://discord.gg/5rJgQTnV4s)
 [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/tjbck)
 
-User-friendly WebUI for LLMs, supported LLM runners include Ollama and OpenAI-compatible APIs. For more information, be sure to check out our [Open WebUI Documentation](https://docs.openwebui.com/).
+Open WebUI is an extensible, feature-rich, and user-friendly self-hosted WebUI designed to operate entirely offline. It supports various LLM runners, including Ollama and OpenAI-compatible APIs. For more information, be sure to check out our [Open WebUI Documentation](https://docs.openwebui.com/).
 
 ![Open WebUI Demo](./demo.gif)
 
@@ -79,6 +79,8 @@ User-friendly WebUI for LLMs, supported LLM runners include Ollama and OpenAI-co
 
 - 🔒 **Backend Reverse Proxy Support**: Bolster security through direct communication between Open WebUI backend and Ollama. This key feature eliminates the need to expose Ollama over LAN. Requests made to the '/ollama/api' route from the web UI are seamlessly redirected to Ollama from the backend, enhancing overall system security.
 
+- 🌐🌍 **Multilingual Support**: Experience Open WebUI in your preferred language with our internationalization (i18n) support. Join us in expanding our supported languages! We're actively seeking contributors!
+
 - 🌟 **Continuous Updates**: We are committed to improving Open WebUI with regular updates and new features.
 
 ## 🔗 Also Check Out Open WebUI Community!

backend/apps/ollama/main.py

@@ -98,13 +98,14 @@ def merge_models_lists(model_lists):
     merged_models = {}
 
     for idx, model_list in enumerate(model_lists):
-        for model in model_list:
-            digest = model["digest"]
-            if digest not in merged_models:
-                model["urls"] = [idx]
-                merged_models[digest] = model
-            else:
-                merged_models[digest]["urls"].append(idx)
+        if model_list is not None:
+            for model in model_list:
+                digest = model["digest"]
+                if digest not in merged_models:
+                    model["urls"] = [idx]
+                    merged_models[digest] = model
+                else:
+                    merged_models[digest]["urls"].append(idx)
 
     return list(merged_models.values())
 
@@ -116,11 +117,10 @@ async def get_all_models():
     print("get_all_models")
     tasks = [fetch_url(f"{url}/api/tags") for url in app.state.OLLAMA_BASE_URLS]
     responses = await asyncio.gather(*tasks)
-    responses = list(filter(lambda x: x is not None, responses))
 
     models = {
         "models": merge_models_lists(
-            map(lambda response: response["models"], responses)
+            map(lambda response: response["models"] if response else None, responses)
         )
     }
 

backend/apps/openai/main.py

@@ -111,6 +111,7 @@ async def speech(request: Request, user=Depends(get_verified_user)):
         headers["Authorization"] = f"Bearer {app.state.OPENAI_API_KEYS[idx]}"
         headers["Content-Type"] = "application/json"
 
+        r = None
         try:
             r = requests.post(
                 url=f"{app.state.OPENAI_API_BASE_URLS[idx]}/audio/speech",
@@ -143,7 +144,9 @@ async def speech(request: Request, user=Depends(get_verified_user)):
                 except:
                     error_detail = f"External: {e}"
 
-            raise HTTPException(status_code=r.status_code, detail=error_detail)
+            raise HTTPException(
+                status_code=r.status_code if r else 500, detail=error_detail
+            )
 
     except ValueError:
         raise HTTPException(status_code=401, detail=ERROR_MESSAGES.OPENAI_NOT_FOUND)
@@ -165,14 +168,15 @@ def merge_models_lists(model_lists):
     merged_list = []
 
     for idx, models in enumerate(model_lists):
-        merged_list.extend(
-            [
-                {**model, "urlIdx": idx}
-                for model in models
-                if "api.openai.com" not in app.state.OPENAI_API_BASE_URLS[idx]
-                or "gpt" in model["id"]
-            ]
-        )
+        if models is not None and "error" not in models:
+            merged_list.extend(
+                [
+                    {**model, "urlIdx": idx}
+                    for model in models
+                    if "api.openai.com" not in app.state.OPENAI_API_BASE_URLS[idx]
+                    or "gpt" in model["id"]
+                ]
+            )
 
     return merged_list
 
@@ -187,15 +191,22 @@ async def get_all_models():
             fetch_url(f"{url}/models", app.state.OPENAI_API_KEYS[idx])
             for idx, url in enumerate(app.state.OPENAI_API_BASE_URLS)
         ]
+
         responses = await asyncio.gather(*tasks)
-        responses = list(
-            filter(lambda x: x is not None and "error" not in x, responses)
-        )
         models = {
             "data": merge_models_lists(
-                list(map(lambda response: response["data"], responses))
+                list(
+                    map(
+                        lambda response: (
+                            response["data"] if "data" in response else None
+                        ),
+                        responses,
+                    )
+                )
             )
         }
+
+        print(models)
         app.state.MODELS = {model["id"]: model for model in models["data"]}
 
         return models
@@ -218,6 +229,9 @@ async def get_models(url_idx: Optional[int] = None, user=Depends(get_current_use
         return models
     else:
         url = app.state.OPENAI_API_BASE_URLS[url_idx]
+
+        r = None
+
         try:
             r = requests.request(method="GET", url=f"{url}/models")
             r.raise_for_status()
@@ -290,6 +304,8 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)):
     headers["Authorization"] = f"Bearer {key}"
     headers["Content-Type"] = "application/json"
 
+    r = None
+
     try:
         r = requests.request(
             method=request.method,
@@ -322,4 +338,6 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)):
             except:
                 error_detail = f"External: {e}"
 
-        raise HTTPException(status_code=r.status_code, detail=error_detail)
+        raise HTTPException(
+            status_code=r.status_code if r else 500, detail=error_detail
+        )

backend/apps/rag/main.py

@@ -400,9 +400,9 @@ def get_loader(filename: str, file_content_type: str, file_path: str):
     elif file_ext in known_source_ext or (
         file_content_type and file_content_type.find("text/") >= 0
     ):
-        loader = TextLoader(file_path)
+        loader = TextLoader(file_path, autodetect_encoding=True)
     else:
-        loader = TextLoader(file_path)
+        loader = TextLoader(file_path, autodetect_encoding=True)
         known_type = False
 
     return loader, known_type

backend/config.py

@@ -250,8 +250,10 @@ OPENAI_API_BASE_URLS = (
     OPENAI_API_BASE_URLS if OPENAI_API_BASE_URLS != "" else OPENAI_API_BASE_URL
 )
 
-OPENAI_API_BASE_URLS = [url.strip() for url in OPENAI_API_BASE_URLS.split(";")]
-
+OPENAI_API_BASE_URLS = [
+    url.strip() if url != "" else "https://api.openai.com/v1"
+    for url in OPENAI_API_BASE_URLS.split(";")
+]
 
 ####################################
 # WEBUI

docs/CONTRIBUTING.md

@@ -50,6 +50,18 @@ We welcome pull requests. Before submitting one, please:
 
 Help us make Open WebUI more accessible by improving documentation, writing tutorials, or creating guides on setting up and optimizing the web UI.
 
+### 🌐 Translations and Internationalization
+
+Help us make Open WebUI available to a wider audience. In this section, we'll guide you through the process of adding new translations to the project.
+
+We use JSON files to store translations. You can find the existing translation files in the `src/lib/i18n/locales` directory. Each directory corresponds to a specific language, for example, `en-US` for English (US), `fr-FR` for French (France) and so on. You can refer to [ISO 639 Language Codes][http://www.lingoes.net/en/translator/langcode.htm] to find the appropriate code for a specific language.
+
+To add a new language:
+
+- Create a new directory in the `src/lib/i18n/locales` path with the appropriate language code as its name. For instance, if you're adding translations for Spanish (Spain), create a new directory named `es-ES`.
+- Copy the American English translation file(s) (from `en-US` directory in `src/lib/i18n/locale`) to this new directory and update the string values in JSON format according to your language. Make sure to preserve the structure of the JSON object.
+- Add the language code and its respective title to languages file at `src/lib/i18n/locales/languages.json`.
+
 ### 🤔 Questions & Feedback
 
 Got questions or feedback? Join our [Discord community](https://discord.gg/5rJgQTnV4s) or open an issue. We're here to help!

i18next-parser.config.ts

+// i18next-parser.config.ts
+import { getLanguages } from './src/lib/i18n/index.ts';
+
+const getLangCodes = async () => {
+	const languages = await getLanguages();
+	return languages.map((l) => l.code);
+};
+
+export default {
+	contextSeparator: '_',
+	createOldCatalogs: false,
+	defaultNamespace: 'translation',
+	defaultValue: '',
+	indentation: 2,
+	keepRemoved: false,
+	keySeparator: false,
+	lexers: {
+		svelte: ['JavascriptLexer'],
+		js: ['JavascriptLexer'],
+		ts: ['JavascriptLexer'],
+
+		default: ['JavascriptLexer']
+	},
+	lineEnding: 'auto',
+	locales: await getLangCodes(),
+	namespaceSeparator: false,
+	output: 'src/lib/i18n/locales/$LOCALE/$NAMESPACE.json',
+	pluralSeparator: '_',
+	input: 'src/**/*.{js,svelte}',
+	sort: true,
+	verbose: true,
+	failOnWarnings: false,
+	failOnUpdate: false,
+	customValueTemplate: null,
+	resetDefaultValueLocale: null,
+	i18nextOptions: null,
+	yamlOptions: null
+};

kubernetes/helm/templates/webui-service.yaml

@@ -4,6 +4,9 @@ metadata:
   name: {{ include "open-webui.name" . }}
   labels:
     {{- include "open-webui.labels" . | nindent 4 }}
+    {{- with .Values.webui.service.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.webui.service.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
@@ -11,14 +14,16 @@ metadata:
 spec:
   selector:
     {{- include "open-webui.selectorLabels" . | nindent 4 }}
-{{- with .Values.webui.service }}
-  type: {{ .type }}
+  type: {{ .Values.webui.service.type | default "ClusterIP" }}
   ports:
   - protocol: TCP
     name: http
-    port: {{ .port }}
+    port: {{ .Values.webui.service.port }}
     targetPort: http
-    {{- if .nodePort }}
-    nodePort: {{ .nodePort | int }}
+    {{- if .Values.webui.service.nodePort }}
+    nodePort: {{ .Values.webui.service.nodePort | int }}
     {{- end }}
-{{- end }}
+  {{- if .Values.webui.service.loadBalancerClass }}
+  loadBalancerClass: {{ .Values.webui.service.loadBalancerClass | quote }}
+  {{- end }}
+

kubernetes/helm/values.yaml

@@ -70,3 +70,5 @@ webui:
     port: 80
     containerPort: 8080
     nodePort: ""
+    labels: {}
+    loadBalancerClass: "" 

package.json

 {
 	"name": "open-webui",
-	"version": "0.1.112",
+	"version": "0.1.113",
 	"private": true,
 	"scripts": {
 		"dev": "vite dev --host",
@@ -13,7 +13,8 @@
 		"lint:types": "npm run check",
 		"lint:backend": "pylint backend/",
 		"format": "prettier --plugin-search-dir --write '**/*.{js,ts,svelte,css,md,html,json}'",
-		"format:backend": "yapf --recursive backend -p -i"
+		"format:backend": "yapf --recursive backend -p -i",
+		"i18n:parse": "i18next --config i18next-parser.config.ts && prettier --write 'src/lib/i18n/**/*.{js,json}'"
 	},
 	"devDependencies": {
 		"@sveltejs/adapter-auto": "^2.0.0",
@@ -27,6 +28,7 @@
 		"eslint": "^8.56.0",
 		"eslint-config-prettier": "^8.5.0",
 		"eslint-plugin-svelte": "^2.30.0",
+		"i18next-parser": "^8.13.0",
 		"postcss": "^8.4.31",
 		"prettier": "^2.8.0",
 		"prettier-plugin-svelte": "^2.10.1",
@@ -42,9 +44,13 @@
 	"dependencies": {
 		"@sveltejs/adapter-node": "^1.3.1",
 		"async": "^3.2.5",
+		"bits-ui": "^0.19.7",
 		"dayjs": "^1.11.10",
 		"file-saver": "^2.0.5",
 		"highlight.js": "^11.9.0",
+		"i18next": "^23.10.0",
+		"i18next-browser-languagedetector": "^7.2.0",
+		"i18next-resources-to-backend": "^1.2.0",
 		"idb": "^7.1.1",
 		"js-sha256": "^0.10.1",
 		"katex": "^0.16.9",

src/app.html

@@ -8,18 +8,35 @@
 		<meta name="robots" content="noindex,nofollow" />
 		<script>
 			// On page load or when changing themes, best to add inline in `head` to avoid FOUC
-			if (
-				localStorage.theme === 'light' ||
-				(!('theme' in localStorage) && window.matchMedia('(prefers-color-scheme: light)').matches)
-			) {
-				document.documentElement.classList.add('light');
-			} else if (localStorage.theme) {
-				localStorage.theme.split(' ').forEach((e) => {
-					document.documentElement.classList.add(e);
+			(() => {
+				if (
+					localStorage.theme === 'light' ||
+					(!('theme' in localStorage) && window.matchMedia('(prefers-color-scheme: light)').matches)
+				) {
+					document.documentElement.classList.add('light');
+				} else if (localStorage.theme && localStorage.theme !== 'system') {
+					localStorage.theme.split(' ').forEach((e) => {
+						document.documentElement.classList.add(e);
+					});
+				} else if (localStorage.theme && localStorage.theme === 'system') {
+					systemTheme = window.matchMedia('(prefers-color-scheme: dark)').matches;
+					document.documentElement.classList.add(systemTheme ? 'dark' : 'light');
+				} else {
+					document.documentElement.classList.add('dark');
+				}
+
+				window.matchMedia('(prefers-color-scheme: dark)').addListener((e) => {
+					if (localStorage.theme === 'system') {
+						if (e.matches) {
+							document.documentElement.classList.add('dark');
+							document.documentElement.classList.remove('light');
+						} else {
+							document.documentElement.classList.add('light');
+							document.documentElement.classList.remove('dark');
+						}
+					}
 				});
-			} else {
-				document.documentElement.classList.add('dark');
-			}
+			})();
 		</script>
 
 		%sveltekit.head%

src/lib/components/AddFilesPlaceholder.svelte

+<script>
+	import { getContext } from 'svelte';
+	const i18n = getContext('i18n');
+</script>
+
 <div class="  text-center text-6xl mb-3">📄</div>
-<div class="text-center dark:text-white text-2xl font-semibold z-50">Add Files</div>
+<div class="text-center dark:text-white text-2xl font-semibold z-50">{$i18n.t('Add Files')}</div>
 
 <slot
 	><div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
-		Drop any files here to add to the conversation
+		{$i18n.t('Drop any files here to add to the conversation')}
 	</div>
 </slot>

src/lib/components/ChangelogModal.svelte

 <script lang="ts">
-	import { onMount } from 'svelte';
+	import { onMount, getContext } from 'svelte';
 	import { Confetti } from 'svelte-confetti';
 
 	import { WEBUI_NAME, config } from '$lib/stores';
@@ -9,6 +9,8 @@
 
 	import Modal from './common/Modal.svelte';
 
+	const i18n = getContext('i18n');
+
 	export let show = false;
 
 	let changelog = null;
@@ -23,7 +25,8 @@
 	<div class="px-5 py-4 dark:text-gray-300">
 		<div class="flex justify-between items-start">
 			<div class="text-xl font-bold">
-				What’s New in {$WEBUI_NAME}
+				{$i18n.t('What’s New in')}
+				{$WEBUI_NAME}
 				<Confetti x={[-1, -0.25]} y={[0, 0.5]} />
 			</div>
 			<button
@@ -45,7 +48,7 @@
 			</button>
 		</div>
 		<div class="flex items-center mt-1">
-			<div class="text-sm dark:text-gray-200">Release Notes</div>
+			<div class="text-sm dark:text-gray-200">{$i18n.t('Release Notes')}</div>
 			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-200 dark:bg-gray-700" />
 			<div class="text-sm dark:text-gray-200">
 				v{WEBUI_VERSION}
@@ -108,7 +111,7 @@
 				}}
 				class=" px-4 py-2 bg-emerald-600 hover:bg-emerald-700 text-gray-100 transition rounded"
 			>
-				<span class="relative">Okay, Let's Go!</span>
+				<span class="relative">{$i18n.t("Okay, Let's Go!")}</span>
 			</button>
 		</div>
 	</div>

src/lib/components/admin/Settings/Database.svelte

 <script lang="ts">
 	import { downloadDatabase } from '$lib/apis/utils';
-	import { onMount } from 'svelte';
+	import { onMount, getContext } from 'svelte';
+
+	const i18n = getContext('i18n');
 
 	export let saveHandler: Function;
 
@@ -17,10 +19,10 @@
 >
 	<div class=" space-y-3 pr-1.5 overflow-y-scroll max-h-80">
 		<div>
-			<div class=" mb-2 text-sm font-medium">Database</div>
+			<div class=" mb-2 text-sm font-medium">{$i18n.t('Database')}</div>
 
 			<div class="  flex w-full justify-between">
-				<!-- <div class=" self-center text-xs font-medium">Allow Chat Deletion</div> -->
+				<!-- <div class=" self-center text-xs font-medium">{$i18n.t('Allow Chat Deletion')}</div> -->
 
 				<button
 					class=" flex rounded-md py-1.5 px-3 w-full hover:bg-gray-200 dark:hover:bg-gray-800 transition"
@@ -46,7 +48,7 @@
 							/>
 						</svg>
 					</div>
-					<div class=" self-center text-sm font-medium">Download Database</div>
+					<div class=" self-center text-sm font-medium">{$i18n.t('Download Database')}</div>
 				</button>
 			</div>
 		</div>