Commit 711e891f authored by Michael Yang's avatar Michael Yang
Browse files

fix resumable downloads

glob returns files in lexical order which is not appropriate when
rebuilding the parts list
parent 090d0842
...@@ -12,6 +12,7 @@ import ( ...@@ -12,6 +12,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
...@@ -39,9 +40,18 @@ type blobDownload struct { ...@@ -39,9 +40,18 @@ type blobDownload struct {
} }
type blobDownloadPart struct { type blobDownloadPart struct {
N int
Offset int64 Offset int64
Size int64 Size int64
Completed int64 Completed int64
*blobDownload `json:"-"`
}
func (p *blobDownloadPart) Name() string {
return strings.Join([]string{
p.blobDownload.Name, "partial", strconv.Itoa(p.N),
}, "-")
} }
func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *RegistryOptions) error { func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *RegistryOptions) error {
...@@ -78,14 +88,10 @@ func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *R ...@@ -78,14 +88,10 @@ func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *R
size = b.Total - offset size = b.Total - offset
} }
partName := b.Name + "-partial-" + strconv.Itoa(len(b.Parts)) if err := b.newPart(offset, size); err != nil {
part := blobDownloadPart{Offset: offset, Size: size}
if err := b.writePart(partName, &part); err != nil {
return err return err
} }
b.Parts = append(b.Parts, &part)
offset += size offset += size
} }
} }
...@@ -161,7 +167,6 @@ func (b *blobDownload) Run(ctx context.Context, requestURL *url.URL, opts *Regis ...@@ -161,7 +167,6 @@ func (b *blobDownload) Run(ctx context.Context, requestURL *url.URL, opts *Regis
func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i int, opts *RegistryOptions) error { func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i int, opts *RegistryOptions) error {
part := b.Parts[i] part := b.Parts[i]
partName := b.File.Name() + "-" + strconv.Itoa(i)
offset := part.Offset + part.Completed offset := part.Offset + part.Completed
w := io.NewOffsetWriter(b.File, offset) w := io.NewOffsetWriter(b.File, offset)
...@@ -181,7 +186,7 @@ func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i ...@@ -181,7 +186,7 @@ func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i
} }
part.Completed += n part.Completed += n
if err := b.writePart(partName, part); err != nil { if err := b.writePart(part.Name(), part); err != nil {
return err return err
} }
...@@ -189,6 +194,16 @@ func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i ...@@ -189,6 +194,16 @@ func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i
return err return err
} }
func (b *blobDownload) newPart(offset, size int64) error {
part := blobDownloadPart{blobDownload: b, Offset: offset, Size: size, N: len(b.Parts)}
if err := b.writePart(part.Name(), &part); err != nil {
return err
}
b.Parts = append(b.Parts, &part)
return nil
}
func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) { func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) {
var part blobDownloadPart var part blobDownloadPart
partFile, err := os.Open(partName) partFile, err := os.Open(partName)
...@@ -201,6 +216,7 @@ func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) { ...@@ -201,6 +216,7 @@ func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) {
return nil, err return nil, err
} }
part.blobDownload = b
return &part, nil return &part, nil
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment