resolved conflicts #2

.env.example

@@ -10,7 +10,8 @@ OPENAI_API_KEY=''
 # DO NOT TRACK
 SCARF_NO_ANALYTICS=true
 DO_NOT_TRACK=true
+ANONYMIZED_TELEMETRY=false
 
 # Use locally bundled version of the LiteLLM cost map json
 # to avoid repetitive startup connections
-LITELLM_LOCAL_MODEL_COST_MAP="True"
+LITELLM_LOCAL_MODEL_COST_MAP="True"
\ No newline at end of file

.github/dependabot.yml

+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/backend"
+  schedule:
+    interval: daily
+    time: "13:00"
+  groups:
+    python-packages:
+      patterns:
+        - "*"

CHANGELOG.md

@@ -5,6 +5,32 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.123] - 2024-05-02
+
+### Added
+
+- **🎨 New Landing Page Design**: Refreshed design for a more modern look and optimized use of screen space.
+- **📹 Youtube RAG Pipeline**: Introduces dedicated RAG pipeline for Youtube videos, enabling interaction with video transcriptions directly.
+- **🔧 Enhanced Admin Panel**: Streamlined user management with options to add users directly or in bulk via CSV import.
+- **👥 '@' Model Integration**: Easily switch to specific models during conversations; old collaborative chat feature phased out.
+- **🌐 Language Enhancements**: Swedish translation added, plus improvements to German, Spanish, and the addition of Doge translation.
+
+### Fixed
+
+- **🗑️ Delete Chat Shortcut**: Addressed issue where shortcut wasn't functioning.
+- **🖼️ Modal Closing Bug**: Resolved unexpected closure of modal when dragging from within.
+- **✏️ Edit Button Styling**: Fixed styling inconsistency with edit buttons.
+- **🌐 Image Generation Compatibility Issue**: Rectified image generation compatibility issue with third-party APIs.
+- **📱 iOS PWA Icon Fix**: Corrected iOS PWA home screen icon shape.
+- **🔍 Scroll Gesture Bug**: Adjusted gesture sensitivity to prevent accidental activation when scrolling through code on mobile; now requires scrolling from the leftmost side to open the sidebar.
+
+### Changed
+
+- **🔄 Unlimited Context Length**: Advanced settings now allow unlimited max context length (previously limited to 16000).
+- **👑 Super Admin Assignment**: The first signup is automatically assigned a super admin role, unchangeable by other admins.
+- **🛡️ Admin User Restrictions**: User action buttons from the admin panel are now disabled for users with admin roles.
+- **🔝 Default Model Selector**: Set as default model option now exclusively available on the landing page.
+
 ## [0.1.122] - 2024-04-27
 
 ### Added

Dockerfile

@@ -51,7 +51,8 @@ ENV OLLAMA_BASE_URL="/ollama" \
 ENV OPENAI_API_KEY="" \
     WEBUI_SECRET_KEY="" \
     SCARF_NO_ANALYTICS=true \
-    DO_NOT_TRACK=true
+    DO_NOT_TRACK=true \
+    ANONYMIZED_TELEMETRY=false
 
 # Use locally bundled version of the LiteLLM cost map json
 # to avoid repetitive startup connections
@@ -74,6 +75,10 @@ ENV HF_HOME="/app/backend/data/cache/embedding/models"
 
 WORKDIR /app/backend
 
+ENV HOME /root
+RUN mkdir -p $HOME/.cache/chroma
+RUN echo -n 00000000-0000-0000-0000-000000000000 > $HOME/.cache/chroma/telemetry_user_id
+
 RUN if [ "$USE_OLLAMA" = "true" ]; then \
         apt-get update && \
         # Install pandoc and netcat
@@ -129,4 +134,4 @@ COPY ./backend .
 
 EXPOSE 8080
 
-CMD [ "bash", "start.sh"]
+CMD [ "bash", "start.sh"]
\ No newline at end of file

backend/apps/images/main.py

@@ -24,6 +24,7 @@ from utils.misc import calculate_sha256
 from typing import Optional
 from pydantic import BaseModel
 from pathlib import Path
+import mimetypes
 import uuid
 import base64
 import json
@@ -315,38 +316,50 @@ class GenerateImageForm(BaseModel):
 
 
 def save_b64_image(b64_str):
-    image_id = str(uuid.uuid4())
-    file_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.png")
-
     try:
-        # Split the base64 string to get the actual image data
-        img_data = base64.b64decode(b64_str)
+        header, encoded = b64_str.split(",", 1)
+        mime_type = header.split(";")[0]
+
+        img_data = base64.b64decode(encoded)
 
-        # Write the image data to a file
+        image_id = str(uuid.uuid4())
+        image_format = mimetypes.guess_extension(mime_type)
+
+        image_filename = f"{image_id}{image_format}"
+        file_path = IMAGE_CACHE_DIR / f"{image_filename}"
         with open(file_path, "wb") as f:
             f.write(img_data)
-
-        return image_id
+        return image_filename
     except Exception as e:
-        log.error(f"Error saving image: {e}")
+        log.exception(f"Error saving image: {e}")
         return None
 
 
 def save_url_image(url):
     image_id = str(uuid.uuid4())
-    file_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.png")
-
     try:
         r = requests.get(url)
         r.raise_for_status()
+        if r.headers["content-type"].split("/")[0] == "image":
+
+            mime_type = r.headers["content-type"]
+            image_format = mimetypes.guess_extension(mime_type)
+
+            if not image_format:
+                raise ValueError("Could not determine image type from MIME type")
 
-        with open(file_path, "wb") as image_file:
-            image_file.write(r.content)
+            file_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}{image_format}")
+            with open(file_path, "wb") as image_file:
+                for chunk in r.iter_content(chunk_size=8192):
+                    image_file.write(chunk)
+            return image_id, image_format
+        else:
+            log.error(f"Url does not point to an image.")
+            return None, None
 
-        return image_id
     except Exception as e:
         log.exception(f"Error saving image: {e}")
-        return None
+        return None, None
 
 
 @app.post("/generations")
@@ -385,8 +398,8 @@ def generate_image(
             images = []
 
             for image in res["data"]:
-                image_id = save_b64_image(image["b64_json"])
-                images.append({"url": f"/cache/image/generations/{image_id}.png"})
+                image_filename = save_b64_image(image["b64_json"])
+                images.append({"url": f"/cache/image/generations/{image_filename}"})
                 file_body_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.json")
 
                 with open(file_body_path, "w") as f:
@@ -422,8 +435,10 @@ def generate_image(
             images = []
 
             for image in res["data"]:
-                image_id = save_url_image(image["url"])
-                images.append({"url": f"/cache/image/generations/{image_id}.png"})
+                image_id, image_format = save_url_image(image["url"])
+                images.append(
+                    {"url": f"/cache/image/generations/{image_id}{image_format}"}
+                )
                 file_body_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.json")
 
                 with open(file_body_path, "w") as f:
@@ -460,8 +475,8 @@ def generate_image(
             images = []
 
             for image in res["images"]:
-                image_id = save_b64_image(image)
-                images.append({"url": f"/cache/image/generations/{image_id}.png"})
+                image_filename = save_b64_image(image)
+                images.append({"url": f"/cache/image/generations/{image_filename}"})
                 file_body_path = IMAGE_CACHE_DIR.joinpath(f"{image_id}.json")
 
                 with open(file_body_path, "w") as f:

backend/apps/rag/main.py

@@ -28,6 +28,7 @@ from langchain_community.document_loaders import (
     UnstructuredXMLLoader,
     UnstructuredRSTLoader,
     UnstructuredExcelLoader,
+    YoutubeLoader,
 )
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 
@@ -181,7 +182,7 @@ class CollectionNameForm(BaseModel):
     collection_name: Optional[str] = "test"
 
 
-class StoreWebForm(CollectionNameForm):
+class UrlForm(CollectionNameForm):
     url: str
 
 
@@ -456,8 +457,32 @@ def query_collection_handler(
         )
 
 
+@app.post("/youtube")
+def store_youtube_video(form_data: UrlForm, user=Depends(get_current_user)):
+    try:
+        loader = YoutubeLoader.from_youtube_url(form_data.url, add_video_info=False)
+        data = loader.load()
+
+        collection_name = form_data.collection_name
+        if collection_name == "":
+            collection_name = calculate_sha256_string(form_data.url)[:63]
+
+        store_data_in_vector_db(data, collection_name, overwrite=True)
+        return {
+            "status": True,
+            "collection_name": collection_name,
+            "filename": form_data.url,
+        }
+    except Exception as e:
+        log.exception(e)
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.DEFAULT(e),
+        )
+
+
 @app.post("/web")
-def store_web(form_data: StoreWebForm, user=Depends(get_current_user)):
+def store_web(form_data: UrlForm, user=Depends(get_current_user)):
     # "https://www.gutenberg.org/files/1727/1727-h/1727-h.htm"
     try:
         loader = get_web_loader(form_data.url)

backend/apps/rag/utils.py

@@ -53,7 +53,7 @@ def query_doc_with_hybrid_search(
     embedding_function,
     k: int,
     reranking_function,
-    r: int,
+    r: float,
 ):
     try:
         collection = CHROMA_CLIENT.get_collection(name=collection_name)
@@ -321,8 +321,12 @@ def rag_messages(
 
     context_string = ""
     for context in relevant_contexts:
-        items = context["documents"][0]
-        context_string += "\n\n".join(items)
+        try:
+            if "documents" in context:
+                items = [item for item in context["documents"][0] if item is not None]
+                context_string += "\n\n".join(items)
+        except Exception as e:
+            log.exception(e)
     context_string = context_string.strip()
 
     ra_content = rag_template(

backend/apps/web/models/auths.py

@@ -89,6 +89,10 @@ class SignupForm(BaseModel):
     profile_image_url: Optional[str] = "/user.png"
 
 
+class AddUserForm(SignupForm):
+    role: Optional[str] = "pending"
+
+
 class AuthsTable:
     def __init__(self, db):
         self.db = db

backend/apps/web/models/users.py

@@ -123,6 +123,13 @@ class UsersTable:
     def get_num_users(self) -> Optional[int]:
         return User.select().count()
 
+    def get_first_user(self) -> UserModel:
+        try:
+            user = User.select().order_by(User.created_at).first()
+            return UserModel(**model_to_dict(user))
+        except:
+            return None
+
     def update_user_role_by_id(self, id: str, role: str) -> Optional[UserModel]:
         try:
             query = User.update(role=role).where(User.id == id)

backend/apps/web/routers/auths.py

 import logging
 
-from fastapi import Request
+from fastapi import Request, UploadFile, File
 from fastapi import Depends, HTTPException, status
 
 from fastapi import APIRouter
 from pydantic import BaseModel
 import re
 import uuid
+import csv
+
 
 from apps.web.models.auths import (
     SigninForm,
     SignupForm,
+    AddUserForm,
     UpdateProfileForm,
     UpdatePasswordForm,
     UserResponse,
@@ -205,6 +208,51 @@ async def signup(request: Request, form_data: SignupForm):
         raise HTTPException(500, detail=ERROR_MESSAGES.DEFAULT(err))
 
 
+############################
+# AddUser
+############################
+
+
+@router.post("/add", response_model=SigninResponse)
+async def add_user(form_data: AddUserForm, user=Depends(get_admin_user)):
+
+    if not validate_email_format(form_data.email.lower()):
+        raise HTTPException(
+            status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.INVALID_EMAIL_FORMAT
+        )
+
+    if Users.get_user_by_email(form_data.email.lower()):
+        raise HTTPException(400, detail=ERROR_MESSAGES.EMAIL_TAKEN)
+
+    try:
+
+        print(form_data)
+        hashed = get_password_hash(form_data.password)
+        user = Auths.insert_new_auth(
+            form_data.email.lower(),
+            hashed,
+            form_data.name,
+            form_data.profile_image_url,
+            form_data.role,
+        )
+
+        if user:
+            token = create_token(data={"id": user.id})
+            return {
+                "token": token,
+                "token_type": "Bearer",
+                "id": user.id,
+                "email": user.email,
+                "name": user.name,
+                "role": user.role,
+                "profile_image_url": user.profile_image_url,
+            }
+        else:
+            raise HTTPException(500, detail=ERROR_MESSAGES.CREATE_USER_ERROR)
+    except Exception as err:
+        raise HTTPException(500, detail=ERROR_MESSAGES.DEFAULT(err))
+
+
 ############################
 # ToggleSignUp
 ############################

backend/apps/web/routers/users.py

@@ -58,7 +58,7 @@ async def update_user_permissions(
 @router.post("/update/role", response_model=Optional[UserModel])
 async def update_user_role(form_data: UserRoleUpdateForm, user=Depends(get_admin_user)):
 
-    if user.id != form_data.id:
+    if user.id != form_data.id and form_data.id != Users.get_first_user().id:
         return Users.update_user_role_by_id(form_data.id, form_data.role)
 
     raise HTTPException(

backend/config.py

@@ -367,6 +367,17 @@ DEFAULT_PROMPT_SUGGESTIONS = (
             "title": ["Show me a code snippet", "of a website's sticky header"],
             "content": "Show me a code snippet of a website's sticky header in CSS and JavaScript.",
         },
+        {
+            "title": [
+                "Explain options trading",
+                "if I'm familiar with buying and selling stocks",
+            ],
+            "content": "Explain options trading in simple terms if I'm familiar with buying and selling stocks.",
+        },
+        {
+            "title": ["Overcome procrastination", "give me tips"],
+            "content": "Could you start by asking me about instances when I procrastinate the most and then give me some suggestions to overcome it?",
+        },
     ]
 )
 

backend/data/config.json

@@ -18,6 +18,18 @@
 			{
 				"title": ["Show me a code snippet", "of a website's sticky header"],
 				"content": "Show me a code snippet of a website's sticky header in CSS and JavaScript."
+			},
+			{
+				"title": ["Explain options trading", "if I'm familiar with buying and selling stocks"],
+				"content": "Explain options trading in simple terms if I'm familiar with buying and selling stocks."
+			},
+			{
+				"title": ["Overcome procrastination", "give me tips"],
+				"content": "Could you start by asking me about instances when I procrastinate the most and then give me some suggestions to overcome it?"
+			},
+			{
+				"title": ["Grammar check", "rewrite it for better readability "],
+				"content": "Check the following sentence for grammar and clarity: \"[sentence]\". Rewrite it for better readability while maintaining its original meaning."
 			}
 		]
 	}

backend/main.py

@@ -311,7 +311,7 @@ async def get_manifest_json():
         "background_color": "#343541",
         "theme_color": "#343541",
         "orientation": "portrait-primary",
-        "icons": [{"src": "/favicon.png", "type": "image/png", "sizes": "844x884"}],
+        "icons": [{"src": "/static/logo.png", "type": "image/png", "sizes": "500x500"}],
     }
 
 

backend/requirements.txt

-fastapi
-uvicorn[standard]
-pydantic
-python-multipart
-
-flask
-flask_cors
-
-python-socketio
-python-jose
-passlib[bcrypt]
-uuid
-
-requests
-aiohttp
-peewee
-peewee-migrate
-psycopg2-binary
-pymysql
-bcrypt
+fastapi==0.109.2
+uvicorn[standard]==0.22.0
+pydantic==2.7.1
+python-multipart==0.0.9
+
+Flask==3.0.3
+Flask-Cors==4.0.0
+
+python-socketio==5.11.2
+python-jose==3.3.0
+passlib[bcrypt]==1.7.4
+uuid==1.30
+
+requests==2.31.0
+aiohttp==3.9.5
+peewee==3.17.3
+peewee-migrate==1.12.2
+psycopg2-binary==2.9.9
+PyMySQL==1.1.0
+bcrypt==4.1.2
 
 litellm==1.35.28
 litellm[proxy]==1.35.28
 
-boto3
-
-argon2-cffi
-apscheduler
-google-generativeai
-
-langchain
-langchain-chroma
-langchain-community
-fake_useragent
-chromadb
-sentence_transformers
-pypdf
-docx2txt
-unstructured
-markdown
-pypandoc
-pandas
-openpyxl
-pyxlsb
-xlrd
-validators
-
-opencv-python-headless
-rapidocr-onnxruntime
-
-fpdf2
-rank_bm25
-
-faster-whisper
-
-PyJWT
-pyjwt[crypto]
-
-black
-langfuse
+boto3==1.34.95
+
+argon2-cffi==23.1.0
+APScheduler==3.10.4
+google-generativeai==0.5.2
+
+langchain==0.1.16
+langchain-community==0.0.34
+langchain-chroma==0.1.0
+
+fake-useragent==1.5.1
+chromadb==0.4.24
+sentence-transformers==2.7.0
+pypdf==4.2.0
+docx2txt==0.8
+unstructured==0.11.8
+Markdown==3.6
+pypandoc==1.13
+pandas==2.2.2
+openpyxl==3.1.2
+pyxlsb==1.0.10
+xlrd==2.0.1
+validators==0.28.1
+
+opencv-python-headless==4.9.0.80
+rapidocr-onnxruntime==1.2.3
+
+fpdf2==2.7.8
+rank-bm25==0.2.2
+
+faster-whisper==1.0.1
+
+PyJWT==2.8.0
+PyJWT[crypto]==2.8.0
+
+black==24.4.2
+langfuse==2.27.3
+youtube-transcript-api

backend/static/user-import.csv

+Name,Email,Password,Role

package-lock.json

 {
 	"name": "open-webui",
-	"version": "0.1.122",
+	"version": "0.1.123",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "open-webui",
-			"version": "0.1.122",
+			"version": "0.1.123",
 			"dependencies": {
 				"@sveltejs/adapter-node": "^1.3.1",
 				"async": "^3.2.5",
 				"bits-ui": "^0.19.7",
 				"dayjs": "^1.11.10",
+				"eventsource-parser": "^1.1.2",
 				"file-saver": "^2.0.5",
 				"highlight.js": "^11.9.0",
 				"i18next": "^23.10.0",
@@ -3167,6 +3168,14 @@
 			"integrity": "sha512-tYUSVOGeQPKt/eC1ABfhHy5Xd96N3oIijJvN3O9+TsC28T5V9yX9oEfEK5faP0EFSNVOG97qtAS68GBrQB2hDg==",
 			"dev": true
 		},
+		"node_modules/eventsource-parser": {
+			"version": "1.1.2",
+			"resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-1.1.2.tgz",
+			"integrity": "sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==",
+			"engines": {
+				"node": ">=14.18"
+			}
+		},
 		"node_modules/execa": {
 			"version": "4.1.0",
 			"resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz",

package.json

 {
 	"name": "open-webui",
-	"version": "0.1.122",
+	"version": "0.1.123",
 	"private": true,
 	"scripts": {
 		"dev": "vite dev --host",
@@ -49,6 +49,7 @@
 		"async": "^3.2.5",
 		"bits-ui": "^0.19.7",
 		"dayjs": "^1.11.10",
+		"eventsource-parser": "^1.1.2",
 		"file-saver": "^2.0.5",
 		"highlight.js": "^11.9.0",
 		"i18next": "^23.10.0",