Skip to content

GitLab

Commit 01811c17 authored by Michael Yang's avatar Michael Yang
Browse files

comments

parent a7248f6e
Show whitespace changes
Inline Side-by-side
Showing with 86 additions and 80 deletions
llm/filetype.go
View file @ 01811c17
...@@ -2,137 +2,139 @@ package llm ...@@ -2,137 +2,139 @@ package llm
import "fmt" import "fmt"
type filetype uint32 type fileType uint32
const ( const (
filetypeF32 filetype = iota fileTypeF32 fileType = iota
filetypeF16 fileTypeF16
filetypeQ4_0 fileTypeQ4_0
filetypeQ4_1 fileTypeQ4_1
filetypeQ4_1_F16 fileTypeQ4_1_F16
filetypeQ8_0 filetype = iota + 2 fileTypeQ4_2 // unused
filetypeQ5_0 fileTypeQ4_3 // unused
filetypeQ5_1 fileTypeQ8_0
filetypeQ2_K fileTypeQ5_0
filetypeQ3_K_S fileTypeQ5_1
filetypeQ3_K_M fileTypeQ2_K
filetypeQ3_K_L fileTypeQ3_K_S
filetypeQ4_K_S fileTypeQ3_K_M
filetypeQ4_K_M fileTypeQ3_K_L
filetypeQ5_K_S fileTypeQ4_K_S
filetypeQ5_K_M fileTypeQ4_K_M
filetypeQ6_K fileTypeQ5_K_S
filetypeIQ2_XXS fileTypeQ5_K_M
filetypeIQ2_XS fileTypeQ6_K
filetypeQ2_K_S fileTypeIQ2_XXS
filetypeQ3_K_XS fileTypeIQ2_XS
filetypeIQ3_XXS fileTypeQ2_K_S
fileTypeQ3_K_XS
fileTypeIQ3_XXS
filetypeUnknown fileTypeUnknown
) )
func ParseFileType(s string) (filetype, error) { func ParseFileType(s string) (fileType, error) {
switch s { switch s {
case "F32": case "F32":
return filetypeF32, nil return fileTypeF32, nil
case "F16": case "F16":
return filetypeF16, nil return fileTypeF16, nil
case "Q4_0": case "Q4_0":
return filetypeQ4_0, nil return fileTypeQ4_0, nil
case "Q4_1": case "Q4_1":
return filetypeQ4_1, nil return fileTypeQ4_1, nil
case "Q4_1_F16": case "Q4_1_F16":
return filetypeQ4_1_F16, nil return fileTypeQ4_1_F16, nil
case "Q8_0": case "Q8_0":
return filetypeQ8_0, nil return fileTypeQ8_0, nil
case "Q5_0": case "Q5_0":
return filetypeQ5_0, nil return fileTypeQ5_0, nil
case "Q5_1": case "Q5_1":
return filetypeQ5_1, nil return fileTypeQ5_1, nil
case "Q2_K": case "Q2_K":
return filetypeQ2_K, nil return fileTypeQ2_K, nil
case "Q3_K_S": case "Q3_K_S":
return filetypeQ3_K_S, nil return fileTypeQ3_K_S, nil
case "Q3_K_M": case "Q3_K_M":
return filetypeQ3_K_M, nil return fileTypeQ3_K_M, nil
case "Q3_K_L": case "Q3_K_L":
return filetypeQ3_K_L, nil return fileTypeQ3_K_L, nil
case "Q4_K_S": case "Q4_K_S":
return filetypeQ4_K_S, nil return fileTypeQ4_K_S, nil
case "Q4_K_M": case "Q4_K_M":
return filetypeQ4_K_M, nil return fileTypeQ4_K_M, nil
case "Q5_K_S": case "Q5_K_S":
return filetypeQ5_K_S, nil return fileTypeQ5_K_S, nil
case "Q5_K_M": case "Q5_K_M":
return filetypeQ5_K_M, nil return fileTypeQ5_K_M, nil
case "Q6_K": case "Q6_K":
return filetypeQ6_K, nil return fileTypeQ6_K, nil
case "IQ2_XXS": case "IQ2_XXS":
return filetypeIQ2_XXS, nil return fileTypeIQ2_XXS, nil
case "IQ2_XS": case "IQ2_XS":
return filetypeIQ2_XS, nil return fileTypeIQ2_XS, nil
case "Q2_K_S": case "Q2_K_S":
return filetypeQ2_K_S, nil return fileTypeQ2_K_S, nil
case "Q3_K_XS": case "Q3_K_XS":
return filetypeQ3_K_XS, nil return fileTypeQ3_K_XS, nil
case "IQ3_XXS": case "IQ3_XXS":
return filetypeIQ3_XXS, nil return fileTypeIQ3_XXS, nil
default: default:
return filetypeUnknown, fmt.Errorf("unknown filetype: %s", s) return fileTypeUnknown, fmt.Errorf("unknown fileType: %s", s)
} }
} }
func (t filetype) String() string { func (t fileType) String() string {
switch t { switch t {
case filetypeF32: case fileTypeF32:
return "F32" return "F32"
case filetypeF16: case fileTypeF16:
return "F16" return "F16"
case filetypeQ4_0: case fileTypeQ4_0:
return "Q4_0" return "Q4_0"
case filetypeQ4_1: case fileTypeQ4_1:
return "Q4_1" return "Q4_1"
case filetypeQ4_1_F16: case fileTypeQ4_1_F16:
return "Q4_1_F16" return "Q4_1_F16"
case filetypeQ8_0: case fileTypeQ8_0:
return "Q8_0" return "Q8_0"
case filetypeQ5_0: case fileTypeQ5_0:
return "Q5_0" return "Q5_0"
case filetypeQ5_1: case fileTypeQ5_1:
return "Q5_1" return "Q5_1"
case filetypeQ2_K: case fileTypeQ2_K:
return "Q2_K" return "Q2_K"
case filetypeQ3_K_S: case fileTypeQ3_K_S:
return "Q3_K_S" return "Q3_K_S"
case filetypeQ3_K_M: case fileTypeQ3_K_M:
return "Q3_K_M" return "Q3_K_M"
case filetypeQ3_K_L: case fileTypeQ3_K_L:
return "Q3_K_L" return "Q3_K_L"
case filetypeQ4_K_S: case fileTypeQ4_K_S:
return "Q4_K_S" return "Q4_K_S"
case filetypeQ4_K_M: case fileTypeQ4_K_M:
return "Q4_K_M" return "Q4_K_M"
case filetypeQ5_K_S: case fileTypeQ5_K_S:
return "Q5_K_S" return "Q5_K_S"
case filetypeQ5_K_M: case fileTypeQ5_K_M:
return "Q5_K_M" return "Q5_K_M"
case filetypeQ6_K: case fileTypeQ6_K:
return "Q6_K" return "Q6_K"
case filetypeIQ2_XXS: case fileTypeIQ2_XXS:
return "IQ2_XXS" return "IQ2_XXS"
case filetypeIQ2_XS: case fileTypeIQ2_XS:
return "IQ2_XS" return "IQ2_XS"
case filetypeQ2_K_S: case fileTypeQ2_K_S:
return "Q2_K_S" return "Q2_K_S"
case filetypeQ3_K_XS: case fileTypeQ3_K_XS:
return "Q3_K_XS" return "Q3_K_XS"
case filetypeIQ3_XXS: case fileTypeIQ3_XXS:
return "IQ3_XXS" return "IQ3_XXS"
default: default:
return "unknown" return "unknown"
} }
} }
func (t filetype) Value() uint32 { func (t fileType) Value() uint32 {
return uint32(t) return uint32(t)
} }
llm/ggml.go
View file @ 01811c17
...@@ -47,7 +47,7 @@ func (kv KV) ParameterCount() uint64 { ...@@ -47,7 +47,7 @@ func (kv KV) ParameterCount() uint64 {
func (kv KV) FileType() string { func (kv KV) FileType() string {
if u64 := kv.u64("general.file_type"); u64 > 0 { if u64 := kv.u64("general.file_type"); u64 > 0 {
return filetype(uint32(u64)).String() return fileType(uint32(u64)).String()
} }
return "unknown" return "unknown"
......
llm/llm.go
View file @ 01811c17
...@@ -20,7 +20,7 @@ func SystemInfo() string { ...@@ -20,7 +20,7 @@ func SystemInfo() string {
return C.GoString(C.llama_print_system_info()) return C.GoString(C.llama_print_system_info())
} }
func Quantize(infile, outfile string, ftype filetype) error { func Quantize(infile, outfile string, ftype fileType) error {
cinfile := C.CString(infile) cinfile := C.CString(infile)
defer C.free(unsafe.Pointer(cinfile)) defer C.free(unsafe.Pointer(cinfile))
......
server/images.go
View file @ 01811c17
...@@ -344,9 +344,9 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m ...@@ -344,9 +344,9 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
switch c.Name { switch c.Name {
case "model", "adapter": case "model", "adapter":
var layers2 *ordered.Map[*Layer, *llm.GGML] var baseLayers *ordered.Map[*Layer, *llm.GGML]
if name := model.ParseName(c.Args, ""); name.IsValid() { if name := model.ParseName(c.Args, ""); name.IsValid() {
layers2, err = parseFromModel(ctx, name, fn) baseLayers, err = parseFromModel(ctx, name, fn)
if err != nil { if err != nil {
return err return err
} }
...@@ -362,14 +362,14 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m ...@@ -362,14 +362,14 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
} }
defer blob.Close() defer blob.Close()
layers2, err = parseFromFile(ctx, blob, fn) baseLayers, err = parseFromFile(ctx, blob, fn)
if err != nil { if err != nil {
return err return err
} }
} else if file, err := os.Open(realpath(modelFileDir, c.Args)); err == nil { } else if file, err := os.Open(realpath(modelFileDir, c.Args)); err == nil {
defer file.Close() defer file.Close()
layers2, err = parseFromFile(ctx, file, fn) baseLayers, err = parseFromFile(ctx, file, fn)
if err != nil { if err != nil {
return err return err
} }
...@@ -381,7 +381,7 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m ...@@ -381,7 +381,7 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
var tempfiles []*os.File var tempfiles []*os.File
// TODO(mxyng): replace with rangefunc // TODO(mxyng): replace with rangefunc
layers2.Items()(func(layer *Layer, ggml *llm.GGML) bool { baseLayers.Items()(func(layer *Layer, ggml *llm.GGML) bool {
if quantization != "" && ggml != nil && ggml.Name() == "gguf" { if quantization != "" && ggml != nil && ggml.Name() == "gguf" {
ftype, err := llm.ParseFileType(quantization) ftype, err := llm.ParseFileType(quantization)
if err != nil { if err != nil {
......
server/model.go
View file @ 01811c17
...@@ -27,7 +27,11 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRe ...@@ -27,7 +27,11 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRe
return nil, err return nil, err
} }
return parseFromModel(ctx, name, fn) modelpath = ParseModelPath(name.DisplayLongest())
manifest, _, err = GetManifest(modelpath)
if err != nil {
return nil, err
}
case err != nil: case err != nil:
return nil, err return nil, err
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment