Initial commit

composite_demo/browser/.gitignore

+# Created by https://www.toptal.com/developers/gitignore/api/node
+# Edit at https://www.toptal.com/developers/gitignore?templates=node
+
+### Node ###
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+### Node Patch ###
+# Serverless Webpack directories
+.webpack/
+
+# Optional stylelint cache
+
+# SvelteKit build / generate output
+.svelte-kit
+
+# End of https://www.toptal.com/developers/gitignore/api/node
\ No newline at end of file

composite_demo/browser/package.json

+{
+  "name": "glm4-browser",
+  "version": "1.0.0",
+  "description": "Browser system for GLM-4",
+  "main": "src/server.ts",
+  "scripts": {
+    "dev": "npx nodemon src/server",
+    "start": "npx ts-node src/server.ts"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "express": "^4.18.3",
+    "jsdom": "^24.0.0",
+    "pnpm": "^9.1.2",
+    "turndown": "^7.1.2",
+    "winston": "^3.11.0"
+  },
+  "devDependencies": {
+    "@types/express": "^4.17.21",
+    "@types/jsdom": "^21.1.6",
+    "@types/node": "^20.11.20",
+    "@types/turndown": "^5.0.4",
+    "nodemon": "^3.1.0",
+    "ts-node": "^10.9.2"
+  }
+}

composite_demo/browser/src/config.ts

+export default {
+    LOG_LEVEL: 'debug',
+
+    BROWSER_TIMEOUT: 10000,
+    BING_SEARCH_API_URL: 'https://api.bing.microsoft.com/',
+    BING_SEARCH_API_KEY: '',
+
+    HOST: 'localhost',
+    PORT: 3000,
+};
\ No newline at end of file

composite_demo/browser/src/server.ts

+import express, { Express, Request, Response } from 'express';
+
+import { SimpleBrowser } from './browser';
+import config from './config';
+import { logger } from './utils';
+
+const session_history: Record<string, SimpleBrowser> = {};
+
+const app: Express = express();
+
+app.use(express.json());
+
+app.post('/', async (req: Request, res: Response) => {
+  const {
+    session_id,
+    action,
+  }: {
+    session_id: string;
+    action: string;
+  } = req.body;
+  logger.info(`session_id: ${session_id}`);
+  logger.info(`action: ${action}`);
+  
+  if (!session_history[session_id]) {
+    session_history[session_id] = new SimpleBrowser();
+  }
+
+  const browser = session_history[session_id];
+
+  try {
+    res.json(await browser.action(action));
+  } catch (err) {
+    logger.error(err);
+    res.status(400).json(err);
+  }
+})
+
+process.on('SIGINT', () => {
+  process.exit(0);
+});
+
+process.on('uncaughtException', e => {
+  logger.error(e);
+});
+
+const { HOST, PORT } = config;
+
+(async () => {
+  app.listen(PORT, HOST, () => {
+    logger.info(`⚡️[server]: Server is running at http://${HOST}:${PORT}`);
+    try {
+      (<any>process).send('ready');
+    } catch (err) {}
+  });
+})();

composite_demo/browser/src/types.ts

+export interface File {
+  id: string;
+  name: string;
+  size: number;
+}
+
+export interface Metadata {
+  files?: File[];
+  reference?: string;
+}
+
+export interface Message {
+  role: 'user' | 'assistant' | 'system' | 'observation';
+  metadata: string;
+  content: string;
+  request_metadata?: Metadata;
+}
+
+export interface ToolObservation {
+  contentType: string;
+  result: string;
+  text?: string;
+  roleMetadata?: string; // metadata for <|observation|>${metadata}
+  metadata: any; // metadata for response
+}

composite_demo/browser/src/utils.ts

+import winston from 'winston';
+
+import config from './config';
+
+export class TimeoutError extends Error {}
+
+const logLevel = config.LOG_LEVEL;
+
+export const logger = winston.createLogger({
+  level: logLevel,
+  format: winston.format.combine(
+    winston.format.colorize(),
+    winston.format.printf(info => {
+      return `${info.level}: ${info.message}`;
+    }),
+  ),
+  transports: [new winston.transports.Console()],
+});
+
+console.log('LOG_LEVEL', logLevel);
+
+export const parseHrtimeToMillisecond = (hrtime: [number, number]): number => {
+    return (hrtime[0] + hrtime[1] / 1e9) * 1000;
+  };
+
+export const promiseWithTime = <T>(
+    promise: Promise<T>
+  ): Promise<{
+    value: T;
+    time: number;
+  }> => {
+    return new Promise((resolve, reject) => {
+      const startTime = process.hrtime();
+      promise
+        .then(value => {
+          resolve({
+            value: value,
+            time: parseHrtimeToMillisecond(process.hrtime(startTime))
+          });
+        })
+        .catch(err => reject(err));
+    });
+  };
+
+export const withTimeout = <T>(
+    millis: number,
+    promise: Promise<T>
+  ): Promise<{
+    value: T;
+    time: number;
+  }> => {
+    const timeout = new Promise<{ value: T; time: number }>((_, reject) =>
+      setTimeout(() => reject(new TimeoutError()), millis)
+    );
+    return Promise.race([promiseWithTime(promise), timeout]);
+  };
\ No newline at end of file

composite_demo/browser/tsconfig.json

+{
+  "compilerOptions": {
+    "target": "es2022",
+    "lib": ["es2022", "dom"],
+    "module": "commonjs",
+    "rootDir": "./",
+    "outDir": "./dist",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+  },
+  "ts-node": {
+    "transpileOnly": true
+  }
+}

composite_demo/requirements.txt

+# Please install the requirments.txt in basic_demo first!
+# use vllm
+# vllm>=0.4.3
+
+accelerate>=0.30.1
+huggingface_hub>=0.19.4
+ipykernel>=6.26.0
+ipython>=8.18.1
+jupyter_client>=8.6.0
+
+langchain>=0.2.1
+langchain-community>=0.2.1
+
+matplotlib>=3.9.0
+pillow>=10.1.0
+pymupdf>=1.24.5
+python-docx>=1.1.2
+python-pptx>=0.6.23
+pyyaml>=6.0.1
+requests>=2.31.0
+sentencepiece
+streamlit>=1.35.0
+tiktoken>=0.7.0
+transformers==4.40.0
+zhipuai>=2.1.0
+

composite_demo/src/client.py

+"""
+
+This is a client part of composite_demo.
+We provide two clients, HFClient and VLLMClient, which are used to interact with the model.
+The HFClient is used to interact with the  transformers backend, and the VLLMClient is used to interact with the VLLM model.
+
+"""
+
+import json
+from collections.abc import Generator
+from copy import deepcopy
+from enum import Enum, auto
+from typing import Protocol
+
+import streamlit as st
+
+from conversation import Conversation, build_system_prompt
+from tools.tool_registry import ALL_TOOLS
+
+
+class ClientType(Enum):
+    HF = auto()
+    VLLM = auto()
+
+
+class Client(Protocol):
+    def __init__(self, model_path: str): ...
+
+    def generate_stream(
+        self,
+        tools: list[dict],
+        history: list[Conversation],
+        **parameters,
+    ) -> Generator[tuple[str | dict, list[dict]]]: ...
+
+
+def process_input(history: list[dict], tools: list[dict]) -> list[dict]:
+    chat_history = []
+    if len(tools) > 0:
+        chat_history.append(
+            {"role": "system", "content": build_system_prompt(list(ALL_TOOLS), tools)}
+        )
+
+    for conversation in history:
+        role = str(conversation.role).removeprefix("<|").removesuffix("|>")
+        item = {
+            "role": role,
+            "content": conversation.content,
+        }
+        if conversation.metadata:
+            item["metadata"] = conversation.metadata
+        # Only append image for user
+        if role == "user" and conversation.image:
+            item["image"] = conversation.image
+        chat_history.append(item)
+
+    return chat_history
+
+
+def process_response(output, history):
+    content = ""
+    history = deepcopy(history)
+    for response in output.split("<|assistant|>"):
+        if "\n" in response:
+            metadata, content = response.split("\n", maxsplit=1)
+        else:
+            metadata, content = "", response
+        if not metadata.strip():
+            content = content.strip()
+            history.append({"role": "assistant", "metadata": metadata, "content": content})
+            content = content.replace("[[训练时间]]", "2023年")
+        else:
+            history.append({"role": "assistant", "metadata": metadata, "content": content})
+            if history[0]["role"] == "system" and "tools" in history[0]:
+                parameters = json.loads(content)
+                content = {"name": metadata.strip(), "parameters": parameters}
+            else:
+                content = {"name": metadata.strip(), "content": content}
+    return content, history
+
+
+# glm-4v-9b is not available in vLLM backend, use HFClient instead.
+@st.cache_resource(max_entries=1, show_spinner="Loading model...")
+def get_client(model_path, typ: ClientType) -> Client:
+    match typ:
+        case ClientType.HF:
+            from clients.hf import HFClient
+
+            return HFClient(model_path)
+        case ClientType.VLLM:
+            try:
+                from clients.vllm import VLLMClient
+            except ImportError as e:
+                e.msg += "; did you forget to install vLLM?"
+                raise
+            return VLLMClient(model_path)
+
+    raise NotImplementedError(f"Client type {typ} is not supported.")