Merge pull request #2476 from open-webui/dev

0.2.0

backend/apps/webui/internal/migrations/009_add_models.py

+"""Peewee migrations -- 009_add_models.py.
+
+Some examples (model - class or model name)::
+
+    > Model = migrator.orm['table_name']            # Return model in current state by name
+    > Model = migrator.ModelClass                   # Return model in current state by name
+
+    > migrator.sql(sql)                             # Run custom SQL
+    > migrator.run(func, *args, **kwargs)           # Run python function with the given args
+    > migrator.create_model(Model)                  # Create a model (could be used as decorator)
+    > migrator.remove_model(model, cascade=True)    # Remove a model
+    > migrator.add_fields(model, **fields)          # Add fields to a model
+    > migrator.change_fields(model, **fields)       # Change fields
+    > migrator.remove_fields(model, *field_names, cascade=True)
+    > migrator.rename_field(model, old_field_name, new_field_name)
+    > migrator.rename_table(model, new_table_name)
+    > migrator.add_index(model, *col_names, unique=False)
+    > migrator.add_not_null(model, *field_names)
+    > migrator.add_default(model, field_name, default)
+    > migrator.add_constraint(model, name, sql)
+    > migrator.drop_index(model, *col_names)
+    > migrator.drop_not_null(model, *field_names)
+    > migrator.drop_constraints(model, *constraints)
+
+"""
+
+from contextlib import suppress
+
+import peewee as pw
+from peewee_migrate import Migrator
+
+
+with suppress(ImportError):
+    import playhouse.postgres_ext as pw_pext
+
+
+def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
+    """Write your migrations here."""
+
+    @migrator.create_model
+    class Model(pw.Model):
+        id = pw.TextField(unique=True)
+        user_id = pw.TextField()
+        base_model_id = pw.TextField(null=True)
+
+        name = pw.TextField()
+
+        meta = pw.TextField()
+        params = pw.TextField()
+
+        created_at = pw.BigIntegerField(null=False)
+        updated_at = pw.BigIntegerField(null=False)
+
+        class Meta:
+            table_name = "model"
+
+
+def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
+    """Write your rollback migrations here."""
+
+    migrator.remove_model("model")

backend/apps/webui/internal/migrations/010_migrate_modelfiles_to_models.py

+"""Peewee migrations -- 009_add_models.py.
+
+Some examples (model - class or model name)::
+
+    > Model = migrator.orm['table_name']            # Return model in current state by name
+    > Model = migrator.ModelClass                   # Return model in current state by name
+
+    > migrator.sql(sql)                             # Run custom SQL
+    > migrator.run(func, *args, **kwargs)           # Run python function with the given args
+    > migrator.create_model(Model)                  # Create a model (could be used as decorator)
+    > migrator.remove_model(model, cascade=True)    # Remove a model
+    > migrator.add_fields(model, **fields)          # Add fields to a model
+    > migrator.change_fields(model, **fields)       # Change fields
+    > migrator.remove_fields(model, *field_names, cascade=True)
+    > migrator.rename_field(model, old_field_name, new_field_name)
+    > migrator.rename_table(model, new_table_name)
+    > migrator.add_index(model, *col_names, unique=False)
+    > migrator.add_not_null(model, *field_names)
+    > migrator.add_default(model, field_name, default)
+    > migrator.add_constraint(model, name, sql)
+    > migrator.drop_index(model, *col_names)
+    > migrator.drop_not_null(model, *field_names)
+    > migrator.drop_constraints(model, *constraints)
+
+"""
+
+from contextlib import suppress
+
+import peewee as pw
+from peewee_migrate import Migrator
+import json
+
+from utils.misc import parse_ollama_modelfile
+
+with suppress(ImportError):
+    import playhouse.postgres_ext as pw_pext
+
+
+def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
+    """Write your migrations here."""
+
+    # Fetch data from 'modelfile' table and insert into 'model' table
+    migrate_modelfile_to_model(migrator, database)
+    # Drop the 'modelfile' table
+    migrator.remove_model("modelfile")
+
+
+def migrate_modelfile_to_model(migrator: Migrator, database: pw.Database):
+    ModelFile = migrator.orm["modelfile"]
+    Model = migrator.orm["model"]
+
+    modelfiles = ModelFile.select()
+
+    for modelfile in modelfiles:
+        # Extract and transform data in Python
+
+        modelfile.modelfile = json.loads(modelfile.modelfile)
+        meta = json.dumps(
+            {
+                "description": modelfile.modelfile.get("desc"),
+                "profile_image_url": modelfile.modelfile.get("imageUrl"),
+                "ollama": {"modelfile": modelfile.modelfile.get("content")},
+                "suggestion_prompts": modelfile.modelfile.get("suggestionPrompts"),
+                "categories": modelfile.modelfile.get("categories"),
+                "user": {**modelfile.modelfile.get("user", {}), "community": True},
+            }
+        )
+
+        info = parse_ollama_modelfile(modelfile.modelfile.get("content"))
+
+        # Insert the processed data into the 'model' table
+        Model.create(
+            id=f"ollama-{modelfile.tag_name}",
+            user_id=modelfile.user_id,
+            base_model_id=info.get("base_model_id"),
+            name=modelfile.modelfile.get("title"),
+            meta=meta,
+            params=json.dumps(info.get("params", {})),
+            created_at=modelfile.timestamp,
+            updated_at=modelfile.timestamp,
+        )
+
+
+def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
+    """Write your rollback migrations here."""
+
+    recreate_modelfile_table(migrator, database)
+    move_data_back_to_modelfile(migrator, database)
+    migrator.remove_model("model")
+
+
+def recreate_modelfile_table(migrator: Migrator, database: pw.Database):
+    query = """
+    CREATE TABLE IF NOT EXISTS modelfile (
+        user_id TEXT,
+        tag_name TEXT,
+        modelfile JSON,
+        timestamp BIGINT
+    )
+    """
+    migrator.sql(query)
+
+
+def move_data_back_to_modelfile(migrator: Migrator, database: pw.Database):
+    Model = migrator.orm["model"]
+    Modelfile = migrator.orm["modelfile"]
+
+    models = Model.select()
+
+    for model in models:
+        # Extract and transform data in Python
+        meta = json.loads(model.meta)
+
+        modelfile_data = {
+            "title": model.name,
+            "desc": meta.get("description"),
+            "imageUrl": meta.get("profile_image_url"),
+            "content": meta.get("ollama", {}).get("modelfile"),
+            "suggestionPrompts": meta.get("suggestion_prompts"),
+            "categories": meta.get("categories"),
+            "user": {k: v for k, v in meta.get("user", {}).items() if k != "community"},
+        }
+
+        # Insert the processed data back into the 'modelfile' table
+        Modelfile.create(
+            user_id=model.user_id,
+            tag_name=model.id,
+            modelfile=modelfile_data,
+            timestamp=model.created_at,
+        )

backend/apps/webui/internal/migrations/011_add_user_settings.py

+"""Peewee migrations -- 002_add_local_sharing.py.
+
+Some examples (model - class or model name)::
+
+    > Model = migrator.orm['table_name']            # Return model in current state by name
+    > Model = migrator.ModelClass                   # Return model in current state by name
+
+    > migrator.sql(sql)                             # Run custom SQL
+    > migrator.run(func, *args, **kwargs)           # Run python function with the given args
+    > migrator.create_model(Model)                  # Create a model (could be used as decorator)
+    > migrator.remove_model(model, cascade=True)    # Remove a model
+    > migrator.add_fields(model, **fields)          # Add fields to a model
+    > migrator.change_fields(model, **fields)       # Change fields
+    > migrator.remove_fields(model, *field_names, cascade=True)
+    > migrator.rename_field(model, old_field_name, new_field_name)
+    > migrator.rename_table(model, new_table_name)
+    > migrator.add_index(model, *col_names, unique=False)
+    > migrator.add_not_null(model, *field_names)
+    > migrator.add_default(model, field_name, default)
+    > migrator.add_constraint(model, name, sql)
+    > migrator.drop_index(model, *col_names)
+    > migrator.drop_not_null(model, *field_names)
+    > migrator.drop_constraints(model, *constraints)
+
+"""
+
+from contextlib import suppress
+
+import peewee as pw
+from peewee_migrate import Migrator
+
+
+with suppress(ImportError):
+    import playhouse.postgres_ext as pw_pext
+
+
+def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
+    """Write your migrations here."""
+
+    # Adding fields settings to the 'user' table
+    migrator.add_fields("user", settings=pw.TextField(null=True))
+
+
+def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
+    """Write your rollback migrations here."""
+
+    # Remove the settings field
+    migrator.remove_fields("user", "settings")

backend/apps/web/internal/migrations/README.md → backend/apps/webui/internal/migrations/README.md

@@ -14,7 +14,7 @@ You will need to create a migration file to ensure that existing databases are u
 2. Make your changes to the models.
 3. From the `backend` directory, run the following command:
    ```bash
-   pw_migrate create --auto --auto-source apps.web.models --database sqlite:///${SQLITE_DB} --directory apps/web/internal/migrations ${MIGRATION_NAME}
+   pw_migrate create --auto --auto-source apps.webui.models --database sqlite:///${SQLITE_DB} --directory apps/web/internal/migrations ${MIGRATION_NAME}
    ```
    - `$SQLITE_DB` should be the path to the database file.
    - `$MIGRATION_NAME` should be a descriptive name for the migration.

backend/apps/web/main.py → backend/apps/webui/main.py

 from fastapi import FastAPI, Depends
 from fastapi.routing import APIRoute
 from fastapi.middleware.cors import CORSMiddleware
-from apps.web.routers import (
+from apps.webui.routers import (
     auths,
     users,
     chats,
     documents,
-    modelfiles,
+    models,
     prompts,
     configs,
     memories,
     utils,
 )
 from config import (
-    WEBUI_VERSION,
+    WEBUI_BUILD_HASH,
     WEBUI_AUTH,
     DEFAULT_MODELS,
     DEFAULT_PROMPT_SUGGESTIONS,
@@ -23,7 +23,9 @@ from config import (
     WEBHOOK_URL,
     WEBUI_AUTH_TRUSTED_EMAIL_HEADER,
     JWT_EXPIRES_IN,
+    WEBUI_BANNERS,
     AppConfig,
+    ENABLE_COMMUNITY_SHARING,
 )
 
 app = FastAPI()
@@ -40,6 +42,11 @@ app.state.config.DEFAULT_PROMPT_SUGGESTIONS = DEFAULT_PROMPT_SUGGESTIONS
 app.state.config.DEFAULT_USER_ROLE = DEFAULT_USER_ROLE
 app.state.config.USER_PERMISSIONS = USER_PERMISSIONS
 app.state.config.WEBHOOK_URL = WEBHOOK_URL
+app.state.config.BANNERS = WEBUI_BANNERS
+
+app.state.config.ENABLE_COMMUNITY_SHARING = ENABLE_COMMUNITY_SHARING
+
+app.state.MODELS = {}
 app.state.AUTH_TRUSTED_EMAIL_HEADER = WEBUI_AUTH_TRUSTED_EMAIL_HEADER
 
 
@@ -56,11 +63,10 @@ app.include_router(users.router, prefix="/users", tags=["users"])
 app.include_router(chats.router, prefix="/chats", tags=["chats"])
 
 app.include_router(documents.router, prefix="/documents", tags=["documents"])
-app.include_router(modelfiles.router, prefix="/modelfiles", tags=["modelfiles"])
+app.include_router(models.router, prefix="/models", tags=["models"])
 app.include_router(prompts.router, prefix="/prompts", tags=["prompts"])
 app.include_router(memories.router, prefix="/memories", tags=["memories"])
 
-
 app.include_router(configs.router, prefix="/configs", tags=["configs"])
 app.include_router(utils.router, prefix="/utils", tags=["utils"])