From 5524c02a58d7c45002ffbbe94d306df5eeba8494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 27 May 2024 12:57:32 +0200 Subject: [PATCH 1/4] Store time of Last-Modified in cache dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Anders F Björklund --- pkg/downloader/downloader.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go index 10c6c5f8d734..1c0306e2e839 100644 --- a/pkg/downloader/downloader.go +++ b/pkg/downloader/downloader.go @@ -44,6 +44,7 @@ const ( type Result struct { Status Status CachePath string // "/Users/foo/Library/Caches/lima/download/by-url-sha256//data" + LastModified string ValidatedDigest bool } @@ -175,7 +176,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, } if o.cacheDir == "" { - if err := downloadHTTP(ctx, localPath, remote, o.description, o.expectedDigest); err != nil { + if err := downloadHTTP(ctx, localPath, "", remote, o.description, o.expectedDigest); err != nil { return nil, err } res := &Result{ @@ -187,6 +188,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, shad := cacheDirectoryPath(o.cacheDir, remote) shadData := filepath.Join(shad, "data") + shadTime := filepath.Join(shad, "time") shadDigest, err := cacheDigestPath(shad, o.expectedDigest) if err != nil { return nil, err @@ -210,6 +212,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, res := &Result{ Status: StatusUsedCache, CachePath: shadData, + LastModified: shadTime, ValidatedDigest: o.expectedDigest != "", } return res, nil @@ -224,7 +227,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, if err := os.WriteFile(shadURL, []byte(remote), 0o644); err != nil { return nil, err } - if err := downloadHTTP(ctx, shadData, remote, o.description, o.expectedDigest); err != nil { + if err := downloadHTTP(ctx, shadData, shadTime, remote, o.description, o.expectedDigest); err != nil { return nil, err } // no need to pass the digest to copyLocal(), as we already verified the digest @@ -239,6 +242,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, res := &Result{ Status: StatusDownloaded, CachePath: shadData, + LastModified: shadTime, ValidatedDigest: o.expectedDigest != "", } return res, nil @@ -266,6 +270,7 @@ func Cached(remote string, opts ...Opt) (*Result, error) { shad := cacheDirectoryPath(o.cacheDir, remote) shadData := filepath.Join(shad, "data") + shadTime := filepath.Join(shad, "time") shadDigest, err := cacheDigestPath(shad, o.expectedDigest) if err != nil { return nil, err @@ -285,6 +290,7 @@ func Cached(remote string, opts ...Opt) (*Result, error) { res := &Result{ Status: StatusUsedCache, CachePath: shadData, + LastModified: shadTime, ValidatedDigest: o.expectedDigest != "", } return res, nil @@ -293,6 +299,7 @@ func Cached(remote string, opts ...Opt) (*Result, error) { // cacheDirectoryPath returns the cache subdirectory path. // - "url" file contains the url // - "data" file contains the data +// - "time" file contains the time (Last-Modified header) func cacheDirectoryPath(cacheDir, remote string) string { return filepath.Join(cacheDir, "download", "by-url-sha256", fmt.Sprintf("%x", sha256.Sum256([]byte(remote)))) } @@ -470,7 +477,7 @@ func validateLocalFileDigest(localPath string, expectedDigest digest.Digest) err return nil } -func downloadHTTP(ctx context.Context, localPath, url, description string, expectedDigest digest.Digest) error { +func downloadHTTP(ctx context.Context, localPath, lastModified, url, description string, expectedDigest digest.Digest) error { if localPath == "" { return fmt.Errorf("downloadHTTP: got empty localPath") } @@ -489,6 +496,12 @@ func downloadHTTP(ctx context.Context, localPath, url, description string, expec if err != nil { return err } + if lastModified != "" { + lm := resp.Header.Get("Last-Modified") + if err := os.WriteFile(lastModified, []byte(lm), 0o644); err != nil { + return err + } + } defer resp.Body.Close() bar, err := progressbar.New(resp.ContentLength) if err != nil { From e8841f64a1c0ded35aeb97a21385aa96764d1e4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 27 May 2024 17:32:51 +0200 Subject: [PATCH 2/4] Store type of Content-Type in cache dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Anders F Björklund --- pkg/downloader/downloader.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go index 1c0306e2e839..3d31a39d9d20 100644 --- a/pkg/downloader/downloader.go +++ b/pkg/downloader/downloader.go @@ -45,6 +45,7 @@ type Result struct { Status Status CachePath string // "/Users/foo/Library/Caches/lima/download/by-url-sha256//data" LastModified string + ContentType string ValidatedDigest bool } @@ -176,7 +177,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, } if o.cacheDir == "" { - if err := downloadHTTP(ctx, localPath, "", remote, o.description, o.expectedDigest); err != nil { + if err := downloadHTTP(ctx, localPath, "", "", remote, o.description, o.expectedDigest); err != nil { return nil, err } res := &Result{ @@ -189,6 +190,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, shad := cacheDirectoryPath(o.cacheDir, remote) shadData := filepath.Join(shad, "data") shadTime := filepath.Join(shad, "time") + shadType := filepath.Join(shad, "type") shadDigest, err := cacheDigestPath(shad, o.expectedDigest) if err != nil { return nil, err @@ -213,6 +215,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, Status: StatusUsedCache, CachePath: shadData, LastModified: shadTime, + ContentType: shadType, ValidatedDigest: o.expectedDigest != "", } return res, nil @@ -227,7 +230,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, if err := os.WriteFile(shadURL, []byte(remote), 0o644); err != nil { return nil, err } - if err := downloadHTTP(ctx, shadData, shadTime, remote, o.description, o.expectedDigest); err != nil { + if err := downloadHTTP(ctx, shadData, shadTime, shadType, remote, o.description, o.expectedDigest); err != nil { return nil, err } // no need to pass the digest to copyLocal(), as we already verified the digest @@ -243,6 +246,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, Status: StatusDownloaded, CachePath: shadData, LastModified: shadTime, + ContentType: shadType, ValidatedDigest: o.expectedDigest != "", } return res, nil @@ -271,6 +275,7 @@ func Cached(remote string, opts ...Opt) (*Result, error) { shad := cacheDirectoryPath(o.cacheDir, remote) shadData := filepath.Join(shad, "data") shadTime := filepath.Join(shad, "time") + shadType := filepath.Join(shad, "type") shadDigest, err := cacheDigestPath(shad, o.expectedDigest) if err != nil { return nil, err @@ -291,6 +296,7 @@ func Cached(remote string, opts ...Opt) (*Result, error) { Status: StatusUsedCache, CachePath: shadData, LastModified: shadTime, + ContentType: shadType, ValidatedDigest: o.expectedDigest != "", } return res, nil @@ -300,6 +306,7 @@ func Cached(remote string, opts ...Opt) (*Result, error) { // - "url" file contains the url // - "data" file contains the data // - "time" file contains the time (Last-Modified header) +// - "type" file contains the type (Content-Type header) func cacheDirectoryPath(cacheDir, remote string) string { return filepath.Join(cacheDir, "download", "by-url-sha256", fmt.Sprintf("%x", sha256.Sum256([]byte(remote)))) } @@ -477,7 +484,7 @@ func validateLocalFileDigest(localPath string, expectedDigest digest.Digest) err return nil } -func downloadHTTP(ctx context.Context, localPath, lastModified, url, description string, expectedDigest digest.Digest) error { +func downloadHTTP(ctx context.Context, localPath, lastModified, contentType, url, description string, expectedDigest digest.Digest) error { if localPath == "" { return fmt.Errorf("downloadHTTP: got empty localPath") } @@ -502,6 +509,12 @@ func downloadHTTP(ctx context.Context, localPath, lastModified, url, description return err } } + if contentType != "" { + ct := resp.Header.Get("Content-Type") + if err := os.WriteFile(contentType, []byte(ct), 0o644); err != nil { + return err + } + } defer resp.Body.Close() bar, err := progressbar.New(resp.ContentLength) if err != nil { From cf20b246891a33a40d67ad15ee67d663a1563f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 27 May 2024 19:52:42 +0200 Subject: [PATCH 3/4] Return contents and not path in result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Anders F Björklund --- pkg/downloader/downloader.go | 26 ++++++++++++++++++++------ pkg/downloader/downloader_test.go | 14 ++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go index 3d31a39d9d20..d75ba72bc2a8 100644 --- a/pkg/downloader/downloader.go +++ b/pkg/downloader/downloader.go @@ -120,6 +120,20 @@ func WithExpectedDigest(expectedDigest digest.Digest) Opt { } } +func readFile(path string) string { + if path == "" { + return "" + } + if _, err := os.Stat(path); err != nil { + return "" + } + b, err := os.ReadFile(path) + if err != nil { + return "" + } + return string(b) +} + // Download downloads the remote resource into the local path. // // Download caches the remote resource if WithCache or WithCacheDir option is specified. @@ -214,8 +228,8 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, res := &Result{ Status: StatusUsedCache, CachePath: shadData, - LastModified: shadTime, - ContentType: shadType, + LastModified: readFile(shadTime), + ContentType: readFile(shadType), ValidatedDigest: o.expectedDigest != "", } return res, nil @@ -245,8 +259,8 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, res := &Result{ Status: StatusDownloaded, CachePath: shadData, - LastModified: shadTime, - ContentType: shadType, + LastModified: readFile(shadTime), + ContentType: readFile(shadType), ValidatedDigest: o.expectedDigest != "", } return res, nil @@ -295,8 +309,8 @@ func Cached(remote string, opts ...Opt) (*Result, error) { res := &Result{ Status: StatusUsedCache, CachePath: shadData, - LastModified: shadTime, - ContentType: shadType, + LastModified: readFile(shadTime), + ContentType: readFile(shadType), ValidatedDigest: o.expectedDigest != "", } return res, nil diff --git a/pkg/downloader/downloader_test.go b/pkg/downloader/downloader_test.go index e5de0d65071c..54c7872e6bef 100644 --- a/pkg/downloader/downloader_test.go +++ b/pkg/downloader/downloader_test.go @@ -10,6 +10,7 @@ import ( "runtime" "strings" "testing" + "time" "github.com/opencontainers/go-digest" "gotest.tools/v3/assert" @@ -25,6 +26,8 @@ func TestDownloadRemote(t *testing.T) { t.Cleanup(ts.Close) dummyRemoteFileURL := ts.URL + "/downloader.txt" const dummyRemoteFileDigest = "sha256:380481d26f897403368be7cb86ca03a4bc14b125bfaf2b93bff809a5a2ad717e" + dummyRemoteFileStat, err := os.Stat(filepath.Join("testdata", "downloader.txt")) + assert.NilError(t, err) t.Run("without cache", func(t *testing.T) { t.Run("without digest", func(t *testing.T) { @@ -105,6 +108,17 @@ func TestDownloadRemote(t *testing.T) { _, err = Cached(dummyRemoteFileURL, WithExpectedDigest(wrongDigest), WithCacheDir(cacheDir)) assert.ErrorContains(t, err, "expected digest") }) + t.Run("metadata", func(t *testing.T) { + _, err := Cached(dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest)) + assert.ErrorContains(t, err, "cache directory to be specified") + + cacheDir := filepath.Join(t.TempDir(), "cache") + r, err := Download(context.Background(), "", dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir)) + assert.NilError(t, err) + assert.Equal(t, StatusDownloaded, r.Status) + assert.Equal(t, dummyRemoteFileStat.ModTime().UTC().Format(time.RFC1123), strings.Replace(r.LastModified, "GMT", "UTC", 1)) + assert.Equal(t, "text/plain; charset=utf-8", r.ContentType) + }) } func TestDownloadLocal(t *testing.T) { From d6bddec8bc6f5b3d438892f27558c11d1e792430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 2 Jun 2024 13:57:49 +0200 Subject: [PATCH 4/4] Use time.Time not string for LastModified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Anders F Björklund --- pkg/downloader/downloader.go | 27 +++++++++++++++++++++++---- pkg/downloader/downloader_test.go | 2 +- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go index d75ba72bc2a8..aa79c68febf8 100644 --- a/pkg/downloader/downloader.go +++ b/pkg/downloader/downloader.go @@ -13,6 +13,7 @@ import ( "path" "path/filepath" "strings" + "time" "github.com/cheggaaa/pb/v3" "github.com/containerd/continuity/fs" @@ -44,7 +45,7 @@ const ( type Result struct { Status Status CachePath string // "/Users/foo/Library/Caches/lima/download/by-url-sha256//data" - LastModified string + LastModified time.Time ContentType string ValidatedDigest bool } @@ -134,6 +135,24 @@ func readFile(path string) string { return string(b) } +func readTime(path string) time.Time { + if path == "" { + return time.Time{} + } + if _, err := os.Stat(path); err != nil { + return time.Time{} + } + b, err := os.ReadFile(path) + if err != nil { + return time.Time{} + } + t, err := time.Parse(http.TimeFormat, string(b)) + if err != nil { + return time.Time{} + } + return t +} + // Download downloads the remote resource into the local path. // // Download caches the remote resource if WithCache or WithCacheDir option is specified. @@ -228,7 +247,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, res := &Result{ Status: StatusUsedCache, CachePath: shadData, - LastModified: readFile(shadTime), + LastModified: readTime(shadTime), ContentType: readFile(shadType), ValidatedDigest: o.expectedDigest != "", } @@ -259,7 +278,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result, res := &Result{ Status: StatusDownloaded, CachePath: shadData, - LastModified: readFile(shadTime), + LastModified: readTime(shadTime), ContentType: readFile(shadType), ValidatedDigest: o.expectedDigest != "", } @@ -309,7 +328,7 @@ func Cached(remote string, opts ...Opt) (*Result, error) { res := &Result{ Status: StatusUsedCache, CachePath: shadData, - LastModified: readFile(shadTime), + LastModified: readTime(shadTime), ContentType: readFile(shadType), ValidatedDigest: o.expectedDigest != "", } diff --git a/pkg/downloader/downloader_test.go b/pkg/downloader/downloader_test.go index 54c7872e6bef..d2111200a617 100644 --- a/pkg/downloader/downloader_test.go +++ b/pkg/downloader/downloader_test.go @@ -116,7 +116,7 @@ func TestDownloadRemote(t *testing.T) { r, err := Download(context.Background(), "", dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir)) assert.NilError(t, err) assert.Equal(t, StatusDownloaded, r.Status) - assert.Equal(t, dummyRemoteFileStat.ModTime().UTC().Format(time.RFC1123), strings.Replace(r.LastModified, "GMT", "UTC", 1)) + assert.Equal(t, dummyRemoteFileStat.ModTime().Truncate(time.Second).UTC(), r.LastModified) assert.Equal(t, "text/plain; charset=utf-8", r.ContentType) }) }