Commit 2c8b680b authored by Bruce MacDonald's avatar Bruce MacDonald
Browse files

use file info for embeddings cache

parent 99b6b600
...@@ -503,9 +503,9 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { ...@@ -503,9 +503,9 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
}() }()
// this will be used to check if we already have embeddings for a file // this will be used to check if we already have embeddings for a file
modelDigest, err := modelDigest(e.model) modelInfo, err := os.Stat(e.model)
if err != nil { if err != nil {
return nil, fmt.Errorf("model digest: %w", err) log.Fatalf("Error getting file info: %s", err)
} }
addedFiles := make(map[string]bool) // keep track of files that have already been added addedFiles := make(map[string]bool) // keep track of files that have already been added
...@@ -521,7 +521,7 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { ...@@ -521,7 +521,7 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
} }
addedFiles[filePath] = true addedFiles[filePath] = true
// check if we already have embeddings for this file path // check if we already have embeddings for this file path
layerIdentifier := fmt.Sprintf("%s:%s", filePath, modelDigest) layerIdentifier := fmt.Sprintf("%s:%s:%s:%d", filePath, e.model, modelInfo.ModTime().Format("2006-01-02 15:04:05"), modelInfo.Size())
digest, _ := GetSHA256Digest(strings.NewReader(layerIdentifier)) digest, _ := GetSHA256Digest(strings.NewReader(layerIdentifier))
existing, err := existingFileEmbeddings(digest) existing, err := existingFileEmbeddings(digest)
if err != nil { if err != nil {
...@@ -591,17 +591,6 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { ...@@ -591,17 +591,6 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
return layers, nil return layers, nil
} }
func modelDigest(modelPath string) (string, error) {
modelFile, err := os.Open(modelPath)
if err != nil {
return "", fmt.Errorf("could not open model blob: %w", err)
}
defer modelFile.Close()
digest, _ := GetSHA256Digest(modelFile)
return digest, nil
}
// existingFileEmbeddings checks if we already have embeddings for a file and loads them into a look-up map // existingFileEmbeddings checks if we already have embeddings for a file and loads them into a look-up map
func existingFileEmbeddings(digest string) (map[string][]float64, error) { func existingFileEmbeddings(digest string) (map[string][]float64, error) {
path, err := GetBlobsPath(digest) path, err := GetBlobsPath(digest)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment