"examples/vscode:/vscode.git/clone" did not exist on "2ab4d0e10f0bfd23c0940539debba205801340b4"
Commit 2c8b680b authored by Bruce MacDonald's avatar Bruce MacDonald
Browse files

use file info for embeddings cache

parent 99b6b600
......@@ -503,9 +503,9 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
}()
// this will be used to check if we already have embeddings for a file
modelDigest, err := modelDigest(e.model)
modelInfo, err := os.Stat(e.model)
if err != nil {
return nil, fmt.Errorf("model digest: %w", err)
log.Fatalf("Error getting file info: %s", err)
}
addedFiles := make(map[string]bool) // keep track of files that have already been added
......@@ -521,7 +521,7 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
}
addedFiles[filePath] = true
// check if we already have embeddings for this file path
layerIdentifier := fmt.Sprintf("%s:%s", filePath, modelDigest)
layerIdentifier := fmt.Sprintf("%s:%s:%s:%d", filePath, e.model, modelInfo.ModTime().Format("2006-01-02 15:04:05"), modelInfo.Size())
digest, _ := GetSHA256Digest(strings.NewReader(layerIdentifier))
existing, err := existingFileEmbeddings(digest)
if err != nil {
......@@ -591,17 +591,6 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
return layers, nil
}
func modelDigest(modelPath string) (string, error) {
modelFile, err := os.Open(modelPath)
if err != nil {
return "", fmt.Errorf("could not open model blob: %w", err)
}
defer modelFile.Close()
digest, _ := GetSHA256Digest(modelFile)
return digest, nil
}
// existingFileEmbeddings checks if we already have embeddings for a file and loads them into a look-up map
func existingFileEmbeddings(digest string) (map[string][]float64, error) {
path, err := GetBlobsPath(digest)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment