subprocess llama.cpp server (#401)

* remove c code * pack llama.cpp * use request context for llama_cpp * let llama_cpp decide the number of threads to use * stop llama runner when app stops * remove sample count and duration metrics * use go generate to get libraries * tmp dir for running llm

subprocess llama.cpp server (#401)
* remove c code * pack llama.cpp * use request context for llama_cpp * let llama_cpp decide the number of threads to use * stop llama runner when app stops * remove sample count and duration metrics * use go generate to get libraries * tmp dir for running llm
42998d79 · Bruce MacDonald · GitHub · f4432e1d · 42998d79 · f4432e1d
Unverified Commit 42998d79 authored Aug 30, 2023 by Bruce MacDonald Committed by GitHub Aug 30, 2023
17 changed files
--- a/llm/ggml_llama.go
+++ b/llm/ggml_llama.go
--- a/llm/k_quants.c
+++ b/llm/k_quants.c
--- a/llm/k_quants.h
+++ b/llm/k_quants.h
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-#include "ggml.h"
-#include <stdint.h>
-#include <assert.h>
-#include <stddef.h>
-// Super-block size
-#ifdef GGML_QKK_64
-#define QK_K 64
-#define K_SCALE_SIZE 4
-#else
-#define QK_K 256
-#define K_SCALE_SIZE 12
-#endif
-#ifndef static_assert
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)
-#define static_assert(cond, msg) _Static_assert(cond, msg)
-#else
-#define static_assert(cond, msg) struct global_scope_noop_trick
-#endif
-#endif
-//
-// Super-block quantization structures
-//
-// 2-bit quantization
-// weight is represented as x = a * q + b
-// 16 blocks of 16 elemenets each
-// Effectively 2.5625 bits per weight
-typedef struct {
-    uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits
-    uint8_t qs[QK_K/4];      // quants
-    ggml_fp16_t d;           // super-block scale for quantized scales
-    ggml_fp16_t dmin;        // super-block scale for quantized mins
-} block_q2_K;
-static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_fp16_t) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");
-// 3-bit quantization
-// weight is represented as x = a * q
-// 16 blocks of 16 elemenets each
-// Effectively 3.4375 bits per weight
-#ifdef GGML_QKK_64
-typedef struct {
-    uint8_t hmask[QK_K/8];     // quants - high bit
-    uint8_t qs[QK_K/4];        // quants - low 2 bits
-    uint8_t scales[2];
-    ggml_fp16_t d;             // super-block scale
-} block_q3_K;
-static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + 2, "wrong q3_K block size/padding");
-#else
-typedef struct {
-    uint8_t hmask[QK_K/8];     // quants - high bit
-    uint8_t qs[QK_K/4];        // quants - low 2 bits
-    uint8_t scales[12];        // scales, quantized with 6 bits
-    ggml_fp16_t d;             // super-block scale
-} block_q3_K;
-static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding");
-#endif
-// 4-bit quantization
-// 16 blocks of 32 elements each
-// weight is represented as x = a * q + b
-// Effectively 4.5 bits per weight
-#ifdef GGML_QKK_64
-typedef struct {
-    ggml_fp16_t d[2];          // super-block scales/mins
-    uint8_t scales[2];         // 4-bit block scales/mins
-    uint8_t qs[QK_K/2];        // 4--bit quants
-} block_q4_K;
-static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + QK_K/2 + 2, "wrong q4_K block size/padding");
-#else
-typedef struct {
-    ggml_fp16_t d;             // super-block scale for quantized scales
-    ggml_fp16_t dmin;          // super-block scale for quantized mins
-    uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
-    uint8_t qs[QK_K/2];        // 4--bit quants
-} block_q4_K;
-static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding");
-#endif
-// 5-bit quantization
-// 16 blocks of 32 elements each
-// weight is represented as x = a * q + b
-// Effectively 5.5 bits per weight
-#ifdef GGML_QKK_64
-typedef struct {
-    ggml_fp16_t d;               // super-block scale
-    int8_t  scales[QK_K/16];     // 8-bit block scales
-    uint8_t qh[QK_K/8];          // quants, high bit
-    uint8_t qs[QK_K/2];          // quants, low 4 bits
-} block_q5_K;
-static_assert(sizeof(block_q5_K) == sizeof(ggml_fp16_t) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding");
-#else
-typedef struct {
-    ggml_fp16_t d;               // super-block scale for quantized scales
-    ggml_fp16_t dmin;            // super-block scale for quantized mins
-    uint8_t scales[K_SCALE_SIZE];   // scales and mins, quantized with 6 bits
-    uint8_t qh[QK_K/8];          // quants, high bit
-    uint8_t qs[QK_K/2];          // quants, low 4 bits
-} block_q5_K;
-static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding");
-#endif
-// 6-bit quantization
-// weight is represented as x = a * q
-// 16 blocks of 16 elemenets each
-// Effectively 6.5625 bits per weight
-typedef struct {
-    uint8_t ql[QK_K/2];      // quants, lower 4 bits
-    uint8_t qh[QK_K/4];      // quants, upper 2 bits
-    int8_t  scales[QK_K/16]; // scales, quantized with 8 bits
-    ggml_fp16_t d;           // super-block scale
-} block_q6_K;
-static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding");
-// This is only used for intermediate quantization and dot products
-typedef struct {
-    float   d;              // delta
-    int8_t  qs[QK_K];       // quants
-    int16_t bsums[QK_K/16]; // sum of quants in groups of 16
-} block_q8_K;
-static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");
-// Quantization
-void quantize_row_q2_K_reference(const float * restrict x, block_q2_K * restrict y, int k);
-void quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict y, int k);
-void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict y, int k);
-void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k);
-void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k);
-void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k);
-void quantize_row_q2_K(const float * restrict x, void * restrict y, int k);
-void quantize_row_q3_K(const float * restrict x, void * restrict y, int k);
-void quantize_row_q4_K(const float * restrict x, void * restrict y, int k);
-void quantize_row_q5_K(const float * restrict x, void * restrict y, int k);
-void quantize_row_q6_K(const float * restrict x, void * restrict y, int k);
-void quantize_row_q8_K(const float * restrict x, void * restrict y, int k);
-// Dequantization
-void dequantize_row_q2_K(const block_q2_K * restrict x, float * restrict y, int k);
-void dequantize_row_q3_K(const block_q3_K * restrict x, float * restrict y, int k);
-void dequantize_row_q4_K(const block_q4_K * restrict x, float * restrict y, int k);
-void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, int k);
-void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int k);
-void dequantize_row_q8_K(const block_q8_K * restrict x, float * restrict y, int k);
-// Dot product
-void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);
-void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);
-void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);
-void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);
-void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);
-// Quantization with histogram collection
-size_t ggml_quantize_q2_K(const float * src, void * dst, int n, int k, int64_t * hist);
-size_t ggml_quantize_q3_K(const float * src, void * dst, int n, int k, int64_t * hist);
-size_t ggml_quantize_q4_K(const float * src, void * dst, int n, int k, int64_t * hist);
-size_t ggml_quantize_q5_K(const float * src, void * dst, int n, int k, int64_t * hist);
-size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist);
--- a/llm/llama-util.h
+++ b/llm/llama-util.h
--- a/llm/llama.cpp
+++ b/llm/llama.cpp
--- a/llm/llama.cpp/generate.go
+++ b/llm/llama.cpp/generate.go
+package llm
+//go:generate git submodule init
+//go:generate git submodule update --force ggml
+//go:generate git -C ggml apply ../ggml_patch/0001-add-detokenize-endpoint.patch
+//go:generate git -C ggml apply ../ggml_patch/0002-34B-model-support.patch
+//go:generate cmake -S ggml -B ggml/build/cpu -DLLAMA_K_QUANTS=on
+//go:generate cmake --build ggml/build/cpu --target server --config Release
--- a/llm/llama.cpp/generate_darwin.go
+++ b/llm/llama.cpp/generate_darwin.go
+//go:build darwin
+// +build darwin
+package llm
+//go:generate git submodule init
+//go:generate git submodule update --force ggml
+//go:generate git -C ggml apply ../ggml_patch/0001-add-detokenize-endpoint.patch
+//go:generate git -C ggml apply ../ggml_patch/0002-34B-model-support.patch
+//go:generate cmake -S ggml -B ggml/build/gpu -DLLAMA_METAL=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on
+//go:generate cmake --build ggml/build/gpu --target server --config Release
--- a/ggml @ 9e232f02
+++ b/ggml @ 9e232f02
+Subproject commit 9e232f0234073358e7031c1b8d7aa45020469a3b
--- a/llm/llama.cpp/ggml_patch/0001-add-detokenize-endpoint.patch
+++ b/llm/llama.cpp/ggml_patch/0001-add-detokenize-endpoint.patch
+From 032ef7ff2423f5117bb59d42fb71be9cebf0a2de Mon Sep 17 00:00:00 2001
+From: Bruce MacDonald <brucewmacdonald@gmail.com>
+Date: Mon, 28 Aug 2023 18:08:12 -0400
+Subject: [PATCH] add detokenize endpoint
+---
+ examples/server/server.cpp | 21 +++++++++++++++++++++
+ 1 file changed, 21 insertions(+)
+diff --git a/examples/server/server.cpp b/examples/server/server.cpp
+index 9966045..5014691 100644
+--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
+@@ -1075,6 +1075,12 @@ static json format_tokenizer_response(const std::vector<llama_token> &tokens)
+         {"tokens", tokens}};
+ }
+static json format_detokenized_response(std::string content)
+{
+    return json{
+        {"content", content}};
+}
+
+ static void parse_options_completion(const json &body, llama_server_context &llama)
+ {
+     gpt_params default_params;
+@@ -1361,6 +1367,21 @@ int main(int argc, char **argv)
+         const json data = format_tokenizer_response(tokens);
+         return res.set_content(data.dump(), "application/json"); });
+    svr.Post("/detokenize", [&llama](const Request &req, Response &res)
+             {
+        auto lock = llama.lock();
+
+        const json body = json::parse(req.body);
+        std::string content;
+        if (body.count("tokens") != 0)
+        {
+            const std::vector<llama_token> tokens = body["tokens"];
+            content = tokens_to_str(llama.ctx, tokens.cbegin(), tokens.cend());
+        }
+
+        const json data = format_detokenized_response(content);
+        return res.set_content(data.dump(), "application/json"); });
+
+     svr.Post("/embedding", [&llama](const Request &req, Response &res)
+              {
+         auto lock = llama.lock();
+-- 
+2.39.2 (Apple Git-143)
--- a/llm/llama.cpp/ggml_patch/0002-34B-model-support.patch
+++ b/llm/llama.cpp/ggml_patch/0002-34B-model-support.patch
+From 6145068a6613c37bb43a7408b5496524bdcfc402 Mon Sep 17 00:00:00 2001
+From: Bruce MacDonald <brucewmacdonald@gmail.com>
+Date: Mon, 28 Aug 2023 18:08:53 -0400
+Subject: [PATCH] 34B model support
+---
+ llama.cpp | 10 ++++++++++
+ 1 file changed, 10 insertions(+)
+diff --git a/llama.cpp b/llama.cpp
+index f2cbe76..62c5cdf 100644
+--- a/llama.cpp
+++ b/llama.cpp
+@@ -79,6 +79,7 @@ enum e_model {
+     MODEL_7B,
+     MODEL_13B,
+     MODEL_30B,
+    MODEL_34B,
+     MODEL_65B,
+     MODEL_70B,
+ };
+@@ -122,6 +123,7 @@ static std::map<e_model, size_t> MEM_REQ_SCRATCH0(int n_ctx)
+         { MODEL_7B,   ((size_t) n_ctx / 16ull + 100ull) * MB },
+         { MODEL_13B,  ((size_t) n_ctx / 12ull + 120ull) * MB },
+         { MODEL_30B,  ((size_t) n_ctx /  9ull + 160ull) * MB },
+        { MODEL_34B,  ((size_t) n_ctx / 9ull + 160ull) * MB },
+         { MODEL_65B,  ((size_t) n_ctx /  6ull + 256ull) * MB }, // guess
+         { MODEL_70B,  ((size_t) n_ctx /  7ull + 164ull) * MB },
+     };
+@@ -135,6 +137,7 @@ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH1()
+         { MODEL_7B,  160ull * MB },
+         { MODEL_13B, 192ull * MB },
+         { MODEL_30B, 256ull * MB },
+        { MODEL_34B, 256ull * MB },
+         { MODEL_65B, 384ull * MB }, // guess
+         { MODEL_70B, 304ull * MB },
+     };
+@@ -149,6 +152,7 @@ static const std::map<e_model, size_t> & MEM_REQ_EVAL()
+         { MODEL_7B,  10ull * MB },
+         { MODEL_13B, 12ull * MB },
+         { MODEL_30B, 16ull * MB },
+        { MODEL_34B, 16ull * MB },
+         { MODEL_65B, 24ull * MB }, // guess
+         { MODEL_70B, 24ull * MB },
+     };
+@@ -164,6 +168,7 @@ static const std::map<e_model, size_t> & VRAM_REQ_SCRATCH_BASE()
+         { MODEL_7B,   512ull * kB },
+         { MODEL_13B,  640ull * kB },
+         { MODEL_30B,  768ull * kB },
+        { MODEL_34B,  768ull * kB },
+         { MODEL_65B, 1280ull * kB },
+         { MODEL_70B, 1280ull * kB },
+     };
+@@ -179,6 +184,7 @@ static const std::map<e_model, size_t> & VRAM_REQ_SCRATCH_PER_CONTEXT()
+         { MODEL_7B,  128ull },
+         { MODEL_13B, 160ull },
+         { MODEL_30B, 208ull },
+        { MODEL_34B, 208ull },
+         { MODEL_65B, 256ull },
+         { MODEL_70B, 256ull },
+     };
+@@ -1027,6 +1033,7 @@ static const char * llama_model_type_name(e_model type) {
+         case MODEL_7B: return "7B";
+         case MODEL_13B: return "13B";
+         case MODEL_30B: return "30B";
+        case MODEL_34B: return "34B";
+         case MODEL_65B: return "65B";
+         case MODEL_70B: return "70B";
+         default: LLAMA_ASSERT(false);
+@@ -1074,6 +1081,7 @@ static void llama_model_load_internal(
+             case 26: model.type = e_model::MODEL_3B; break;
+             case 32: model.type = e_model::MODEL_7B; break;
+             case 40: model.type = e_model::MODEL_13B; break;
+            case 48: model.type = e_model::MODEL_34B; break;
+             case 60: model.type = e_model::MODEL_30B; break;
+             case 80: model.type = e_model::MODEL_65B; break;
+             default:
+@@ -1094,6 +1102,8 @@ static void llama_model_load_internal(
+             LLAMA_LOG_WARN("%s: warning: assuming 70B model based on GQA == %d\n", __func__, n_gqa);
+             model.type = e_model::MODEL_70B;
+             hparams.f_ffn_mult = 1.3f; // from the params.json of the 70B model
+        } else if (model.type == e_model::MODEL_34B && n_gqa == 8) {
+            hparams.f_ffn_mult = 1.0f; // from the params.json of the 34B model
+         }
+         hparams.rope_freq_base  = rope_freq_base;
+-- 
+2.39.2 (Apple Git-143)
--- a/llm/llama.go
+++ b/llm/llama.go
--- a/llm/llama.h
+++ b/llm/llama.h
--- a/llm/llama_darwin.go
+++ b/llm/llama_darwin.go
-package llm
-import (
-	"bytes"
-	"crypto/sha256"
-	"errors"
-	"io"
-	"log"
-	"os"
-	"path/filepath"
-)
-func init() {
-	if err := initBackend(); err != nil {
-		log.Printf("WARNING: GPU could not be initialized correctly: %v", err)
-		log.Printf("WARNING: falling back to CPU")
-	}
-}
-func initBackend() error {
-	exec, err := os.Executable()
-	if err != nil {
-		return err
-	}
-	exec, err = filepath.EvalSymlinks(exec)
-	if err != nil {
-		return err
-	}
-	metal := filepath.Join(filepath.Dir(exec), "ggml-metal.metal")
-	fi, err := os.Stat(metal)
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-	if fi != nil {
-		actual, err := os.Open(metal)
-		if err != nil {
-			return err
-		}
-		defer actual.Close()
-		actualSum := sha256.New()
-		if _, err := io.Copy(actualSum, actual); err != nil {
-			return err
-		}
-		expect, err := fs.Open("ggml-metal.metal")
-		if err != nil {
-			return err
-		}
-		expectSum := sha256.New()
-		if _, err := io.Copy(expectSum, expect); err != nil {
-			return err
-		}
-		if bytes.Equal(actualSum.Sum(nil), expectSum.Sum(nil)) {
-			return nil
-		}
-	}
-	dst, err := os.Create(filepath.Join(filepath.Dir(exec), "ggml-metal.metal"))
-	if err != nil {
-		return err
-	}
-	defer dst.Close()
-	src, err := fs.Open("ggml-metal.metal")
-	if err != nil {
-		return err
-	}
-	defer src.Close()
-	if _, err := io.Copy(dst, src); err != nil {
-		return err
-	}
-	return nil
-}
--- a/llm/llm.go
+++ b/llm/llm.go
 package llm
 import (
+	"context"
 	"fmt"
 	"log"
 	"os"
@@ -11,12 +12,13 @@ import (
 )
 type LLM interface {
-	Predict([]int, string, func(api.GenerateResponse)) error
+	Predict(context.Context, []int, string, func(api.GenerateResponse)) error
-	Embedding(string) ([]float64, error)
+	Embedding(context.Context, string) ([]float64, error)
-	Encode(string) []int
+	Encode(context.Context, string) ([]int, error)
-	Decode(...int) string
+	Decode(context.Context, []int) (string, error)
 	SetOptions(api.Options)
 	Close()
+	Ping(context.Context) error
 }
 func New(model string, adapters []string, opts api.Options) (LLM, error) {
@@ -75,7 +77,7 @@ func New(model string, adapters []string, opts api.Options) (LLM, error) {
 	switch ggml.ModelFamily() {
 	case ModelFamilyLlama:
-		return newLlama(model, adapters, opts)
+		return newLlama(model, adapters, ggmlRunner(), opts)
 	default:
 		return nil, fmt.Errorf("unknown ggml type: %s", ggml.ModelFamily())
 	}

--- a/llm/update-llama-cpp.sh
+++ b/llm/update-llama-cpp.sh
-#!/bin/sh
-set -eu
-status() { echo >&2 ">>> $*"; }
-error() { status "ERROR $*"; }
-usage() {
-    echo "usage: $(basename $0) /path/to/repo"
-    exit 1
-}
-OUT=$(dirname $0)
-while getopts "hC:" OPTION; do
-    case $OPTION in
-        C) OUT=$OPTARG ;;
-        *) usage ;;
-    esac
-done
-shift $(( $OPTIND - 1 ))
-[ $# -eq 1 ] || usage
-status "updating source..."
-cp -a "$1"/*.{c,h,cpp,m,metal,cu} "$OUT"
-status "removing incompatible files..."
-rm -f "$OUT"/build-info.h
-SHA1=$(git -C $1 rev-parse @)
-LICENSE=$(mktemp)
-cleanup() {
-    rm -f $LICENSE
-}
-trap cleanup 0
-cat <<EOF | sed 's/ *$//' >$LICENSE
-/**
- * llama.cpp - git $SHA1
- *
-$(sed 's/^/ * /' <$1/LICENSE)
- */
-EOF
-for IN in $OUT/*.{c,h,cpp,m,metal,cu}; do
-    TMP=$(mktemp)
-    status "updating license $IN"
-    cat $LICENSE $IN >$TMP
-    mv $TMP $IN
-done
-touchup() {
-    local CONSTRAINT=$1 && shift
-    for IN in $*; do
-        status "touching up $IN..."
-        TMP=$(mktemp)
-        {
-            echo "//go:build $CONSTRAINT"
-            echo
-        } | cat - $IN >$TMP
-        mv $TMP $IN
-    done
-}
-touchup darwin $OUT/ggml-metal.*
-touchup mpi $OUT/ggml-mpi.*
-touchup opencl $OUT/ggml-opencl.*
--- a/server/images.go
+++ b/server/images.go
--- a/server/routes.go
+++ b/server/routes.go