Merge pull request #167 from jmorganca/decode-ggml

partial decode ggml bin for more info

llama/llama_darwin.go → llm/llama_darwin.go

-package llama
+package llm
 
 import (
 	"bytes"

llm/llm.go

+package llm
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+type LLM interface {
+	Predict([]int, string, func(api.GenerateResponse)) error
+	Embedding(string) ([]float64, error)
+	Encode(string) []int
+	Decode(...int) string
+	SetOptions(api.Options)
+	Close()
+}
+
+func New(model string, opts api.Options) (LLM, error) {
+	if _, err := os.Stat(model); err != nil {
+		return nil, err
+	}
+
+	f, err := os.Open(model)
+	if err != nil {
+		return nil, err
+	}
+
+	ggml, err := DecodeGGML(f, ModelFamilyLlama)
+	if err != nil {
+		return nil, err
+	}
+
+	switch ggml.ModelFamily {
+	case ModelFamilyLlama:
+		return newLlama(model, opts)
+	default:
+		return nil, fmt.Errorf("unknown ggml type: %s", ggml.ModelFamily)
+	}
+}

llama/utils.go → llm/utils.go

-package llama
+package llm
 
 import (
 	"fmt"

server/images.go

@@ -19,7 +19,7 @@ import (
 	"strings"
 
 	"github.com/jmorganca/ollama/api"
-	"github.com/jmorganca/ollama/llama"
+	"github.com/jmorganca/ollama/llm"
 	"github.com/jmorganca/ollama/parser"
 	"github.com/jmorganca/ollama/vector"
 )
@@ -99,9 +99,14 @@ type LayerReader struct {
 }
 
 type ConfigV2 struct {
+	ModelFamily llm.ModelFamily `json:"model_family"`
+	ModelType   llm.ModelType   `json:"model_type"`
+	FileType    llm.FileType    `json:"file_type"`
+	RootFS      RootFS          `json:"rootfs"`
+
+	// required by spec
 	Architecture string `json:"architecture"`
 	OS           string `json:"os"`
-	RootFS       RootFS `json:"rootfs"`
 }
 
 type RootFS struct {
@@ -246,6 +251,11 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api
 		return err
 	}
 
+	config := ConfigV2{
+		Architecture: "amd64",
+		OS:           "linux",
+	}
+
 	var layers []*LayerReader
 	params := make(map[string][]string)
 	embed := EmbeddingParams{fn: fn, opts: api.DefaultOptions()}
@@ -284,6 +294,18 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api
 					}
 					defer file.Close()
 
+					ggml, err := llm.DecodeGGML(file, llm.ModelFamilyLlama)
+					if err != nil {
+						return err
+					}
+
+					config.ModelFamily = ggml.ModelFamily
+					config.ModelType = ggml.ModelType
+					config.FileType = ggml.FileType
+
+					// reset the file
+					file.Seek(0, io.SeekStart)
+
 					l, err := CreateLayer(file)
 					if err != nil {
 						return fmt.Errorf("failed to create layer: %v", err)
@@ -292,6 +314,7 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api
 					layers = append(layers, l)
 				}
 			}
+
 			if mf != nil {
 				log.Printf("manifest = %#v", mf)
 				for _, l := range mf.Layers {
@@ -321,7 +344,7 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api
 			layers = append(layers, layer)
 		case "template", "system", "prompt":
 			fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)})
-			// remove the prompt layer if one exists
+			// remove the layer if one exists
 			mediaType := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name)
 			layers = removeLayerFromLayers(layers, mediaType)
 
@@ -383,7 +406,7 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api
 
 	// Create a layer for the config object
 	fn(api.ProgressResponse{Status: "creating config layer"})
-	cfg, err := createConfigLayer(digests)
+	cfg, err := createConfigLayer(config, digests)
 	if err != nil {
 		return err
 	}
@@ -430,13 +453,13 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
 		}
 
 		e.opts.EmbeddingOnly = true
-		llm, err := llama.New(e.model, e.opts)
+		llmModel, err := llm.New(e.model, e.opts)
 		if err != nil {
 			return nil, fmt.Errorf("load model to generate embeddings: %v", err)
 		}
 		defer func() {
-			if llm != nil {
-				llm.Close()
+			if llmModel != nil {
+				llmModel.Close()
 			}
 		}()
 
@@ -480,7 +503,7 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
 						Total:     len(data) - 1,
 						Completed: i,
 					})
-					embed, err := llm.Embedding(d)
+					embed, err := llmModel.Embedding(d)
 					if err != nil {
 						log.Printf("failed to generate embedding for '%s' line %d: %v", filePath, i+1, err)
 						continue
@@ -676,7 +699,7 @@ func getLayerDigests(layers []*LayerReader) ([]string, error) {
 // CreateLayer creates a Layer object from a given file
 func CreateLayer(f io.ReadSeeker) (*LayerReader, error) {
 	digest, size := GetSHA256Digest(f)
-	f.Seek(0, 0)
+	f.Seek(0, io.SeekStart)
 
 	layer := &LayerReader{
 		Layer: Layer{
@@ -768,10 +791,6 @@ func DeleteModel(name string) error {
 		return err
 	}
 
-	if err != nil {
-		return err
-	}
-
 	// only delete the files which are still in the deleteMap
 	for k, v := range deleteMap {
 		if v {
@@ -970,15 +989,10 @@ func pullModelManifest(mp ModelPath, regOpts *RegistryOptions) (*ManifestV2, err
 	return m, err
 }
 
-func createConfigLayer(layers []string) (*LayerReader, error) {
-	// TODO change architecture and OS
-	config := ConfigV2{
-		Architecture: "arm64",
-		OS:           "linux",
-		RootFS: RootFS{
+func createConfigLayer(config ConfigV2, layers []string) (*LayerReader, error) {
+	config.RootFS = RootFS{
 		Type:    "layers",
 		DiffIDs: layers,
-		},
 	}
 
 	configJSON, err := json.Marshal(config)

server/routes.go

@@ -21,14 +21,14 @@ import (
 	"gonum.org/v1/gonum/mat"
 
 	"github.com/jmorganca/ollama/api"
-	"github.com/jmorganca/ollama/llama"
+	"github.com/jmorganca/ollama/llm"
 	"github.com/jmorganca/ollama/vector"
 )
 
 var loaded struct {
 	mu sync.Mutex
 
-	llm        *llama.LLM
+	llm        llm.LLM
 	Embeddings []vector.Embedding
 
 	expireAt    time.Time
@@ -63,11 +63,16 @@ func load(model *Model, reqOpts map[string]interface{}, sessionDuration time.Dur
 			loaded.Embeddings = model.Embeddings
 		}
 
-		llm, err := llama.New(model.ModelPath, opts)
+		llmModel, err := llm.New(model.ModelPath, opts)
 		if err != nil {
 			return err
 		}
 
+		// set cache values before modifying opts
+		loaded.llm = llmModel
+		loaded.digest = model.Digest
+		loaded.options = opts
+
 		if opts.NumKeep < 0 {
 			promptWithSystem, err := model.Prompt(api.GenerateRequest{}, "")
 			if err != nil {
@@ -79,15 +84,13 @@ func load(model *Model, reqOpts map[string]interface{}, sessionDuration time.Dur
 				return err
 			}
 
-			tokensWithSystem := llm.Encode(promptWithSystem)
-			tokensNoSystem := llm.Encode(promptNoSystem)
+			tokensWithSystem := llmModel.Encode(promptWithSystem)
+			tokensNoSystem := llmModel.Encode(promptNoSystem)
 
-			llm.NumKeep = len(tokensWithSystem) - len(tokensNoSystem) + 1
-		}
+			opts.NumKeep = len(tokensWithSystem) - len(tokensNoSystem) + 1
 
-		loaded.llm = llm
-		loaded.digest = model.Digest
-		loaded.options = opts
+			llmModel.SetOptions(opts)
+		}
 	}
 	loaded.expireAt = time.Now().Add(sessionDuration)