Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
ollama
Commits
2c8b680b
"examples/vscode:/vscode.git/clone" did not exist on "2ab4d0e10f0bfd23c0940539debba205801340b4"
Commit
2c8b680b
authored
Aug 14, 2023
by
Bruce MacDonald
Browse files
use file info for embeddings cache
parent
99b6b600
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
3 additions
and
14 deletions
+3
-14
server/images.go
server/images.go
+3
-14
No files found.
server/images.go
View file @
2c8b680b
...
...
@@ -503,9 +503,9 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
}()
// this will be used to check if we already have embeddings for a file
model
Digest
,
err
:=
modelDiges
t
(
e
.
model
)
model
Info
,
err
:=
os
.
Sta
t
(
e
.
model
)
if
err
!=
nil
{
return
nil
,
fmt
.
Errorf
(
"model digest
: %
w
"
,
err
)
log
.
Fatalf
(
"Error getting file info
: %
s
"
,
err
)
}
addedFiles
:=
make
(
map
[
string
]
bool
)
// keep track of files that have already been added
...
...
@@ -521,7 +521,7 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
}
addedFiles
[
filePath
]
=
true
// check if we already have embeddings for this file path
layerIdentifier
:=
fmt
.
Sprintf
(
"%s:%s"
,
filePath
,
model
Digest
)
layerIdentifier
:=
fmt
.
Sprintf
(
"%s:%s
:%s:%d
"
,
filePath
,
e
.
model
,
modelInfo
.
ModTime
()
.
Format
(
"2006-01-02 15:04:05"
),
modelInfo
.
Size
()
)
digest
,
_
:=
GetSHA256Digest
(
strings
.
NewReader
(
layerIdentifier
))
existing
,
err
:=
existingFileEmbeddings
(
digest
)
if
err
!=
nil
{
...
...
@@ -591,17 +591,6 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
return
layers
,
nil
}
func
modelDigest
(
modelPath
string
)
(
string
,
error
)
{
modelFile
,
err
:=
os
.
Open
(
modelPath
)
if
err
!=
nil
{
return
""
,
fmt
.
Errorf
(
"could not open model blob: %w"
,
err
)
}
defer
modelFile
.
Close
()
digest
,
_
:=
GetSHA256Digest
(
modelFile
)
return
digest
,
nil
}
// existingFileEmbeddings checks if we already have embeddings for a file and loads them into a look-up map
func
existingFileEmbeddings
(
digest
string
)
(
map
[
string
][]
float64
,
error
)
{
path
,
err
:=
GetBlobsPath
(
digest
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment