Merge pull request #2002 from Yanyutin753/addAudio

🤩 Added custom openai tts models and role variables

backend/apps/audio/main.py

@@ -43,6 +43,8 @@ from config import (
     DEVICE_TYPE,
     AUDIO_OPENAI_API_BASE_URL,
     AUDIO_OPENAI_API_KEY,
+    AUDIO_OPENAI_API_MODEL,
+    AUDIO_OPENAI_API_VOICE,
 )
 
 log = logging.getLogger(__name__)
@@ -60,6 +62,8 @@ app.add_middleware(
 
 app.state.OPENAI_API_BASE_URL = AUDIO_OPENAI_API_BASE_URL
 app.state.OPENAI_API_KEY = AUDIO_OPENAI_API_KEY
+app.state.OPENAI_API_MODEL = AUDIO_OPENAI_API_MODEL
+app.state.OPENAI_API_VOICE = AUDIO_OPENAI_API_VOICE
 
 # setting device type for whisper model
 whisper_device_type = DEVICE_TYPE if DEVICE_TYPE and DEVICE_TYPE == "cuda" else "cpu"
@@ -72,6 +76,8 @@ SPEECH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
 class OpenAIConfigUpdateForm(BaseModel):
     url: str
     key: str
+    model: str
+    speaker: str
 
 
 @app.get("/config")
@@ -79,6 +85,8 @@ async def get_openai_config(user=Depends(get_admin_user)):
     return {
         "OPENAI_API_BASE_URL": app.state.OPENAI_API_BASE_URL,
         "OPENAI_API_KEY": app.state.OPENAI_API_KEY,
+        "OPENAI_API_MODEL": app.state.OPENAI_API_MODEL,
+        "OPENAI_API_VOICE": app.state.OPENAI_API_VOICE,
     }
 
 
@@ -91,11 +99,15 @@ async def update_openai_config(
 
     app.state.OPENAI_API_BASE_URL = form_data.url
     app.state.OPENAI_API_KEY = form_data.key
+    app.state.OPENAI_API_MODEL = form_data.model
+    app.state.OPENAI_API_VOICE = form_data.speaker
 
     return {
         "status": True,
         "OPENAI_API_BASE_URL": app.state.OPENAI_API_BASE_URL,
         "OPENAI_API_KEY": app.state.OPENAI_API_KEY,
+        "OPENAI_API_MODEL": app.state.OPENAI_API_MODEL,
+        "OPENAI_API_VOICE": app.state.OPENAI_API_VOICE,
     }
 
 

backend/config.py

@@ -586,6 +586,8 @@ IMAGE_GENERATION_MODEL = os.getenv("IMAGE_GENERATION_MODEL", "")
 
 AUDIO_OPENAI_API_BASE_URL = os.getenv("AUDIO_OPENAI_API_BASE_URL", OPENAI_API_BASE_URL)
 AUDIO_OPENAI_API_KEY = os.getenv("AUDIO_OPENAI_API_KEY", OPENAI_API_KEY)
+AUDIO_OPENAI_API_MODEL = os.getenv("AUDIO_OPENAI_API_MODEL", "tts-1")
+AUDIO_OPENAI_API_VOICE = os.getenv("AUDIO_OPENAI_API_VOICE", "alloy")
 
 ####################################
 # LiteLLM

src/lib/apis/audio/index.ts

@@ -30,6 +30,8 @@ export const getAudioConfig = async (token: string) => {
 type OpenAIConfigForm = {
 	url: string;
 	key: string;
+	model: string;
+	speaker: string;
 };
 
 export const updateAudioConfig = async (token: string, payload: OpenAIConfigForm) => {
@@ -95,7 +97,8 @@ export const transcribeAudio = async (token: string, file: File) => {
 export const synthesizeOpenAISpeech = async (
 	token: string = '',
 	speaker: string = 'alloy',
-	text: string = ''
+	text: string = '',
+	model: string = 'tts-1'
 ) => {
 	let error = null;
 
@@ -106,7 +109,7 @@ export const synthesizeOpenAISpeech = async (
 			'Content-Type': 'application/json'
 		},
 		body: JSON.stringify({
-			model: 'tts-1',
+			model: model,
 			input: text,
 			voice: speaker
 		})

src/lib/apis/openai/index.ts

@@ -239,7 +239,8 @@ export const generateOpenAIChatCompletion = async (
 export const synthesizeOpenAISpeech = async (
 	token: string = '',
 	speaker: string = 'alloy',
-	text: string = ''
+	text: string = '',
+	model: string = 'tts-1'
 ) => {
 	let error = null;
 
@@ -250,7 +251,7 @@ export const synthesizeOpenAISpeech = async (
 			'Content-Type': 'application/json'
 		},
 		body: JSON.stringify({
-			model: 'tts-1',
+			model: model,
 			input: text,
 			voice: speaker
 		})

src/lib/components/chat/Messages/ResponseMessage.svelte

@@ -227,7 +227,8 @@
 					const res = await synthesizeOpenAISpeech(
 						localStorage.token,
 						$settings?.audio?.speaker,
-						sentence
+						sentence,
+						$settings?.audio?.model
 					).catch((error) => {
 						toast.error(error);
 

src/lib/components/chat/Settings/Audio.svelte

@@ -26,6 +26,8 @@
 
 	let voices = [];
 	let speaker = '';
+	let models = [];
+	let model = '';
 
 	const getOpenAIVoices = () => {
 		voices = [
@@ -38,6 +40,10 @@
 		];
 	};
 
+	const getOpenAIVoicesModel = () => {
+		models = [{ name: 'tts-1' }, { name: 'tts-1-hd' }];
+	};
+
 	const getWebAPIVoices = () => {
 		const getVoicesLoop = setInterval(async () => {
 			voices = await speechSynthesis.getVoices();
@@ -78,12 +84,16 @@
 		if (TTSEngine === 'openai') {
 			const res = await updateAudioConfig(localStorage.token, {
 				url: OpenAIUrl,
-				key: OpenAIKey
+				key: OpenAIKey,
+				model: model,
+				speaker: speaker
 			});
 
 			if (res) {
 				OpenAIUrl = res.OPENAI_API_BASE_URL;
 				OpenAIKey = res.OPENAI_API_KEY;
+				model = res.OPENAI_API_MODEL;
+				speaker = res.OPENAI_API_VOICE;
 			}
 		}
 	};
@@ -98,9 +108,11 @@
 		STTEngine = settings?.audio?.STTEngine ?? '';
 		TTSEngine = settings?.audio?.TTSEngine ?? '';
 		speaker = settings?.audio?.speaker ?? '';
+		model = settings?.audio?.model ?? '';
 
 		if (TTSEngine === 'openai') {
 			getOpenAIVoices();
+			getOpenAIVoicesModel();
 		} else {
 			getWebAPIVoices();
 		}
@@ -111,6 +123,8 @@
 			if (res) {
 				OpenAIUrl = res.OPENAI_API_BASE_URL;
 				OpenAIKey = res.OPENAI_API_KEY;
+				model = res.OPENAI_API_MODEL;
+				speaker = res.OPENAI_API_VOICE;
 			}
 		}
 	});
@@ -126,7 +140,8 @@
 			audio: {
 				STTEngine: STTEngine !== '' ? STTEngine : undefined,
 				TTSEngine: TTSEngine !== '' ? TTSEngine : undefined,
-				speaker: speaker !== '' ? speaker : undefined
+				speaker: speaker !== '' ? speaker : undefined,
+				model: model !== '' ? model : undefined
 			}
 		});
 		dispatch('save');
@@ -215,6 +230,7 @@
 							if (e.target.value === 'openai') {
 								getOpenAIVoices();
 								speaker = 'alloy';
+								model = 'tts-1';
 							} else {
 								getWebAPIVoices();
 								speaker = '';
@@ -307,6 +323,25 @@
 					</div>
 				</div>
 			</div>
+			<div>
+				<div class=" mb-2.5 text-sm font-medium">{$i18n.t('Set Model')}</div>
+				<div class="flex w-full">
+					<div class="flex-1">
+						<input
+							list="model-list"
+							class="w-full rounded-lg py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-850 outline-none"
+							bind:value={model}
+							placeholder="Select a model"
+						/>
+
+						<datalist id="model-list">
+							{#each models as model}
+								<option value={model.name} />
+							{/each}
+						</datalist>
+					</div>
+				</div>
+			</div>
 		{/if}
 	</div>
 

src/lib/i18n/locales/ar-BH/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "ضبط الخطوات",
 	"Set Title Auto-Generation Model": "قم بتعيين نموذج إنشاء العنوان تلقائيًا",
 	"Set Voice": "ضبط الصوت",
+	"Set Model": "ضبط النموذج",
 	"Settings": "الاعدادات",
 	"Settings saved successfully!": "تم حفظ الاعدادات بنجاح",
 	"Share": "كشاركة",

src/lib/i18n/locales/bg-BG/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "Задай Стъпки",
 	"Set Title Auto-Generation Model": "Задай Модел за Автоматично Генериране на Заглавие",
 	"Set Voice": "Задай Глас",
+	"Set Model": "Задай Модел",
 	"Settings": "Настройки",
 	"Settings saved successfully!": "Настройките са запазени успешно!",
 	"Share": "",

src/lib/i18n/locales/bn-BD/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "পরবর্তী ধাপসমূহ",
 	"Set Title Auto-Generation Model": "শিরোনাম অটোজেনারেশন মডেন নির্ধারণ করুন",
 	"Set Voice": "কন্ঠস্বর নির্ধারণ করুন",
+	"Set Model": "মডেল নির্ধারণ করুন",
 	"Settings": "সেটিংসমূহ",
 	"Settings saved successfully!": "সেটিংগুলো সফলভাবে সংরক্ষিত হয়েছে",
 	"Share": "",

src/lib/i18n/locales/ca-ES/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "Estableix Passos",
 	"Set Title Auto-Generation Model": "Estableix Model d'Auto-Generació de Títol",
 	"Set Voice": "Estableix Veu",
+	"Set Model": "Estableix Model",
 	"Settings": "Configuracions",
 	"Settings saved successfully!": "Configuracions guardades amb èxit!",
 	"Share": "",

src/lib/i18n/locales/de-DE/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "Schritte festlegen",
 	"Set Title Auto-Generation Model": "Modell für automatische Titelgenerierung festlegen",
 	"Set Voice": "Stimme festlegen",
+	"Set Model": "Modell festlegen",
 	"Settings": "Einstellungen",
 	"Settings saved successfully!": "Einstellungen erfolgreich gespeichert!",
 	"Share": "Teilen",

src/lib/i18n/locales/dg-DG/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "Set Steps so many steps",
 	"Set Title Auto-Generation Model": "Set Title Auto-Generation Model very auto-generate",
 	"Set Voice": "Set Voice so speak",
+	"Set Model": "Set Model so speak",
 	"Settings": "Settings much settings",
 	"Settings saved successfully!": "Settings saved successfully! Very success!",
 	"Share": "",

src/lib/i18n/locales/en-GB/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "",
 	"Set Title Auto-Generation Model": "",
 	"Set Voice": "",
+	"Set Model": "",
 	"Settings": "",
 	"Settings saved successfully!": "",
 	"Share": "",

src/lib/i18n/locales/en-US/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "",
 	"Set Title Auto-Generation Model": "",
 	"Set Voice": "",
+	"Set Model": "",
 	"Settings": "",
 	"Settings saved successfully!": "",
 	"Share": "",

src/lib/i18n/locales/es-ES/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "Establecer Pasos",
 	"Set Title Auto-Generation Model": "Establecer modelo de generación automática de títulos",
 	"Set Voice": "Establecer la voz",
+	"Set Model": "Establecer el modelo",
 	"Settings": "Configuración",
 	"Settings saved successfully!": "¡Configuración guardada exitosamente!",
 	"Share": "",

src/lib/i18n/locales/fa-IR/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "تنظیم گام\u200cها",
 	"Set Title Auto-Generation Model": "تنظیم مدل تولید خودکار عنوان",
 	"Set Voice": "تنظیم صدا",
+	"Set Model": "تنظیم مدل",
 	"Settings": "تنظیمات",
 	"Settings saved successfully!": "تنظیمات با موفقیت ذخیره شد!",
 	"Share": "",

src/lib/i18n/locales/fr-CA/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "Définir les étapes",
 	"Set Title Auto-Generation Model": "Définir le modèle de génération automatique de titre",
 	"Set Voice": "Définir la voix",
+	"Set Model": "Configurer le modèle",
 	"Settings": "Paramètres",
 	"Settings saved successfully!": "Paramètres enregistrés avec succès !",
 	"Share": "",

src/lib/i18n/locales/fr-FR/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "Définir les étapes",
 	"Set Title Auto-Generation Model": "Définir le modèle de génération automatique de titre",
 	"Set Voice": "Définir la voix",
+	"Set Model": "Définir le modèle",
 	"Settings": "Paramètres",
 	"Settings saved successfully!": "Paramètres enregistrés avec succès !",
 	"Share": "",

src/lib/i18n/locales/it-IT/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "Imposta passaggi",
 	"Set Title Auto-Generation Model": "Imposta modello di generazione automatica del titolo",
 	"Set Voice": "Imposta voce",
+	"Set Model": "Imposta modello",
 	"Settings": "Impostazioni",
 	"Settings saved successfully!": "Impostazioni salvate con successo!",
 	"Share": "",

src/lib/i18n/locales/ja-JP/translation.json

@@ -391,6 +391,7 @@
 	"Set Steps": "ステップを設定",
 	"Set Title Auto-Generation Model": "タイトル自動生成モデルを設定",
 	"Set Voice": "音声を設定",
+	"Set Model": "モデルを設定",
 	"Settings": "設定",
 	"Settings saved successfully!": "設定が正常に保存されました！",
 	"Share": "",