Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
ollama
Commits
123a722a
Unverified
Commit
123a722a
authored
Jun 26, 2024
by
Michael Yang
Committed by
GitHub
Jun 26, 2024
Browse files
zip: prevent extracting files into parent dirs (#5314)
parent
4d311eb7
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
133 additions
and
22 deletions
+133
-22
cmd/cmd.go
cmd/cmd.go
+3
-3
server/model.go
server/model.go
+38
-19
server/model_test.go
server/model_test.go
+92
-0
No files found.
cmd/cmd.go
View file @
123a722a
...
@@ -162,9 +162,6 @@ func tempZipFiles(path string) (string, error) {
...
@@ -162,9 +162,6 @@ func tempZipFiles(path string) (string, error) {
}
}
defer
tempfile
.
Close
()
defer
tempfile
.
Close
()
zipfile
:=
zip
.
NewWriter
(
tempfile
)
defer
zipfile
.
Close
()
detectContentType
:=
func
(
path
string
)
(
string
,
error
)
{
detectContentType
:=
func
(
path
string
)
(
string
,
error
)
{
f
,
err
:=
os
.
Open
(
path
)
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
if
err
!=
nil
{
...
@@ -233,6 +230,9 @@ func tempZipFiles(path string) (string, error) {
...
@@ -233,6 +230,9 @@ func tempZipFiles(path string) (string, error) {
files
=
append
(
files
,
tks
...
)
files
=
append
(
files
,
tks
...
)
}
}
zipfile
:=
zip
.
NewWriter
(
tempfile
)
defer
zipfile
.
Close
()
for
_
,
file
:=
range
files
{
for
_
,
file
:=
range
files
{
f
,
err
:=
os
.
Open
(
file
)
f
,
err
:=
os
.
Open
(
file
)
if
err
!=
nil
{
if
err
!=
nil
{
...
...
server/model.go
View file @
123a722a
...
@@ -11,6 +11,7 @@ import (
...
@@ -11,6 +11,7 @@ import (
"net/http"
"net/http"
"os"
"os"
"path/filepath"
"path/filepath"
"strings"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/convert"
"github.com/ollama/ollama/convert"
...
@@ -77,62 +78,80 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRe
...
@@ -77,62 +78,80 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRe
return
layers
,
nil
return
layers
,
nil
}
}
func
parse
FromZipFile
(
_
context
.
Context
,
file
*
os
.
File
,
digest
string
,
fn
func
(
api
.
ProgressResponse
))
(
layers
[]
*
layerGGML
,
err
error
)
{
func
extract
FromZipFile
(
p
string
,
file
*
os
.
File
,
fn
func
(
api
.
ProgressResponse
))
error
{
stat
,
err
:=
file
.
Stat
()
stat
,
err
:=
file
.
Stat
()
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
err
}
}
r
,
err
:=
zip
.
NewReader
(
file
,
stat
.
Size
())
r
,
err
:=
zip
.
NewReader
(
file
,
stat
.
Size
())
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
err
}
tempdir
,
err
:=
os
.
MkdirTemp
(
filepath
.
Dir
(
file
.
Name
()),
""
)
if
err
!=
nil
{
return
nil
,
err
}
}
defer
os
.
RemoveAll
(
tempdir
)
fn
(
api
.
ProgressResponse
{
Status
:
"unpacking model metadata"
})
fn
(
api
.
ProgressResponse
{
Status
:
"unpacking model metadata"
})
for
_
,
f
:=
range
r
.
File
{
for
_
,
f
:=
range
r
.
File
{
n
:=
filepath
.
Join
(
p
,
f
.
Name
)
if
!
strings
.
HasPrefix
(
n
,
p
)
{
slog
.
Warn
(
"skipped extracting file outside of context"
,
"name"
,
f
.
Name
)
continue
}
if
err
:=
os
.
MkdirAll
(
filepath
.
Dir
(
n
),
0
o750
);
err
!=
nil
{
return
err
}
// TODO(mxyng): this should not write out all files to disk
// TODO(mxyng): this should not write out all files to disk
outfile
,
err
:=
os
.
Create
(
filepath
.
Join
(
tempdir
,
f
.
Name
)
)
outfile
,
err
:=
os
.
Create
(
n
)
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
err
}
}
defer
outfile
.
Close
()
defer
outfile
.
Close
()
infile
,
err
:=
f
.
Open
()
infile
,
err
:=
f
.
Open
()
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
err
}
}
defer
infile
.
Close
()
defer
infile
.
Close
()
if
_
,
err
=
io
.
Copy
(
outfile
,
infile
);
err
!=
nil
{
if
_
,
err
=
io
.
Copy
(
outfile
,
infile
);
err
!=
nil
{
return
nil
,
err
return
err
}
}
if
err
:=
outfile
.
Close
();
err
!=
nil
{
if
err
:=
outfile
.
Close
();
err
!=
nil
{
return
nil
,
err
return
err
}
}
if
err
:=
infile
.
Close
();
err
!=
nil
{
if
err
:=
infile
.
Close
();
err
!=
nil
{
return
nil
,
err
return
err
}
}
}
}
mf
,
err
:=
convert
.
GetModelFormat
(
tempdir
)
return
nil
}
func
parseFromZipFile
(
_
context
.
Context
,
file
*
os
.
File
,
digest
string
,
fn
func
(
api
.
ProgressResponse
))
(
layers
[]
*
layerGGML
,
err
error
)
{
tempDir
,
err
:=
os
.
MkdirTemp
(
filepath
.
Dir
(
file
.
Name
()),
""
)
if
err
!=
nil
{
return
nil
,
err
}
defer
os
.
RemoveAll
(
tempDir
)
if
err
:=
extractFromZipFile
(
tempDir
,
file
,
fn
);
err
!=
nil
{
return
nil
,
err
}
mf
,
err
:=
convert
.
GetModelFormat
(
tempDir
)
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
nil
,
err
}
}
params
,
err
:=
mf
.
GetParams
(
temp
d
ir
)
params
,
err
:=
mf
.
GetParams
(
temp
D
ir
)
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
nil
,
err
}
}
mArch
,
err
:=
mf
.
GetModelArch
(
""
,
temp
d
ir
,
params
)
mArch
,
err
:=
mf
.
GetModelArch
(
""
,
temp
D
ir
,
params
)
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
nil
,
err
}
}
...
@@ -150,7 +169,7 @@ func parseFromZipFile(_ context.Context, file *os.File, digest string, fn func(a
...
@@ -150,7 +169,7 @@ func parseFromZipFile(_ context.Context, file *os.File, digest string, fn func(a
// TODO(mxyng): this should write directly into a layer
// TODO(mxyng): this should write directly into a layer
// e.g. NewLayer(arch.Reader(), "application/vnd.ollama.image.model")
// e.g. NewLayer(arch.Reader(), "application/vnd.ollama.image.model")
temp
,
err
:=
os
.
CreateTemp
(
temp
d
ir
,
"fp16"
)
temp
,
err
:=
os
.
CreateTemp
(
temp
D
ir
,
"fp16"
)
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
nil
,
err
}
}
...
...
server/model_test.go
0 → 100644
View file @
123a722a
package
server
import
(
"archive/zip"
"bytes"
"io"
"os"
"path/filepath"
"slices"
"testing"
"github.com/ollama/ollama/api"
)
func
createZipFile
(
t
*
testing
.
T
,
name
string
)
*
os
.
File
{
t
.
Helper
()
f
,
err
:=
os
.
CreateTemp
(
t
.
TempDir
(),
""
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
zf
:=
zip
.
NewWriter
(
f
)
defer
zf
.
Close
()
zh
,
err
:=
zf
.
CreateHeader
(
&
zip
.
FileHeader
{
Name
:
name
})
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
if
_
,
err
:=
io
.
Copy
(
zh
,
bytes
.
NewReader
([]
byte
(
""
)));
err
!=
nil
{
t
.
Fatal
(
err
)
}
return
f
}
func
TestExtractFromZipFile
(
t
*
testing
.
T
)
{
cases
:=
[]
struct
{
name
string
expect
[]
string
}{
{
name
:
"good"
,
expect
:
[]
string
{
"good"
},
},
{
name
:
filepath
.
Join
(
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
".."
,
"bad"
),
},
}
for
_
,
tt
:=
range
cases
{
t
.
Run
(
tt
.
name
,
func
(
t
*
testing
.
T
)
{
f
:=
createZipFile
(
t
,
tt
.
name
)
defer
f
.
Close
()
tempDir
:=
t
.
TempDir
()
if
err
:=
extractFromZipFile
(
tempDir
,
f
,
func
(
api
.
ProgressResponse
)
{});
err
!=
nil
{
t
.
Fatal
(
err
)
}
var
matches
[]
string
if
err
:=
filepath
.
Walk
(
tempDir
,
func
(
p
string
,
fi
os
.
FileInfo
,
err
error
)
error
{
if
err
!=
nil
{
return
err
}
if
!
fi
.
IsDir
()
{
matches
=
append
(
matches
,
p
)
}
return
nil
});
err
!=
nil
{
t
.
Fatal
(
err
)
}
var
actual
[]
string
for
_
,
match
:=
range
matches
{
rel
,
err
:=
filepath
.
Rel
(
tempDir
,
match
)
if
err
!=
nil
{
t
.
Error
(
err
)
}
actual
=
append
(
actual
,
rel
)
}
if
!
slices
.
Equal
(
actual
,
tt
.
expect
)
{
t
.
Fatalf
(
"expected %d files, got %d"
,
len
(
tt
.
expect
),
len
(
matches
))
}
})
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment