Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 1433bee

Browse filesBrowse files
crazy-maxcrazy-max
committed
image: refresh identity cache entries before expiry
Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>
1 parent 59bf6b3 commit 1433bee
Copy full SHA for 1433bee

3 files changed

+309-11Lines changed: 309 additions & 11 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎daemon/containerd/image_identity.go‎

Copy file name to clipboardExpand all lines: daemon/containerd/image_identity.go
+230-10Lines changed: 230 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"net"
88
"net/url"
99
"slices"
10+
"sort"
1011
"strings"
1112
"sync"
1213
"time"
@@ -26,13 +27,19 @@ import (
2627
"github.com/opencontainers/go-digest"
2728
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
2829
"github.com/pkg/errors"
30+
"golang.org/x/sync/errgroup"
2931
"golang.org/x/sync/singleflight"
3032
)
3133

3234
const (
3335
imageIdentityCacheTTL = 48 * time.Hour
3436
imageIdentityErrorCacheTTL = 15 * time.Minute
3537
imageIdentityWarmupTimeout = 2 * time.Minute
38+
39+
imageIdentityRefreshConcurrency = 4
40+
imageIdentityRefreshTimeout = 2 * time.Minute
41+
imageIdentityRefreshInterval = 30 * time.Minute
42+
imageIdentityRefreshAhead = imageIdentityRefreshInterval + imageIdentityRefreshTimeout
3643
)
3744

3845
type imageIdentityCacheEntry = identitycache.Entry
@@ -42,6 +49,10 @@ type imageIdentityState struct {
4249
cacheMu sync.Mutex
4350
cache map[string]imageIdentityCacheEntry
4451
cacheStore identitycache.Backend
52+
53+
stopRefreshOnce sync.Once
54+
stopRefresh chan struct{}
55+
refreshDone chan struct{}
4556
}
4657

4758
func (i *ImageService) imageIdentity(ctx context.Context, desc ocispec.Descriptor, multi *multiPlatformSummary) (*imagetypes.Identity, error) {
@@ -74,16 +85,7 @@ func (i *ImageService) imageIdentityWithCachePolicy(ctx context.Context, desc oc
7485
}
7586

7687
computedSignature, hasTransientVerificationError := i.computeSignatureIdentity(ctx, desc, multi)
77-
ttl := imageIdentityCacheTTL
78-
if hasTransientVerificationError {
79-
// signature verification errors can be temporary (e.g. no network),
80-
// so cache these for a shorter period
81-
ttl = imageIdentityErrorCacheTTL
82-
}
83-
if err := i.updateImageIdentityCache(ctx, cacheKey, computedSignature, ttl); err != nil {
84-
log.G(ctx).WithError(err).WithField("image", desc.Digest).Debug("failed to update image identity cache entry")
85-
}
86-
88+
computedSignature = i.cacheComputedSignatureIdentity(ctx, cacheKey, desc, computedSignature, hasTransientVerificationError)
8789
return computedSignature, nil
8890
})
8991
if err != nil {
@@ -292,6 +294,19 @@ func (i *ImageService) updateImageIdentityCache(ctx context.Context, cacheKey st
292294
return i.identity.cacheStore.Store(ctx, cacheKey, entry, now)
293295
}
294296

297+
func (i *ImageService) cacheComputedSignatureIdentity(ctx context.Context, cacheKey string, desc ocispec.Descriptor, computedSignature *imagetypes.SignatureIdentity, hasTransientVerificationError bool) *imagetypes.SignatureIdentity {
298+
ttl := imageIdentityCacheTTL
299+
if hasTransientVerificationError {
300+
// signature verification errors can be temporary (e.g. no network),
301+
// so cache these for a shorter period
302+
ttl = imageIdentityErrorCacheTTL
303+
}
304+
if err := i.updateImageIdentityCache(ctx, cacheKey, computedSignature, ttl); err != nil {
305+
log.G(ctx).WithError(err).WithField("image", desc.Digest).Debug("failed to update image identity cache entry")
306+
}
307+
return computedSignature
308+
}
309+
295310
func cloneSignatureIdentity(s *imagetypes.SignatureIdentity) *imagetypes.SignatureIdentity {
296311
if s == nil {
297312
return nil
@@ -314,6 +329,211 @@ func pruneImageIdentityCacheEntries(entries map[string]imageIdentityCacheEntry,
314329
}
315330
}
316331

332+
func imageIdentityCacheEntryNeedsRefresh(entry imageIdentityCacheEntry, now time.Time) bool {
333+
if entry.ExpiresAt.IsZero() {
334+
return false
335+
}
336+
return !entry.ExpiresAt.After(now.Add(imageIdentityRefreshAhead))
337+
}
338+
339+
func (i *ImageService) imageIdentityCacheKeysToRefresh(ctx context.Context, now time.Time) ([]string, error) {
340+
seen := map[string]struct{}{}
341+
342+
i.identity.cacheMu.Lock()
343+
for key, entry := range i.identity.cache {
344+
if imageIdentityCacheEntryNeedsRefresh(entry, now) {
345+
seen[key] = struct{}{}
346+
}
347+
}
348+
i.identity.cacheMu.Unlock()
349+
350+
if i.identity.cacheStore != nil {
351+
if err := i.identity.cacheStore.Walk(ctx, now, func(cacheKey string, entry imageIdentityCacheEntry) error {
352+
if imageIdentityCacheEntryNeedsRefresh(entry, now) {
353+
seen[cacheKey] = struct{}{}
354+
}
355+
return nil
356+
}); err != nil {
357+
return nil, err
358+
}
359+
}
360+
361+
out := make([]string, 0, len(seen))
362+
for key := range seen {
363+
out = append(out, key)
364+
}
365+
sort.Strings(out)
366+
return out, nil
367+
}
368+
369+
func (i *ImageService) refreshImageIdentityCache(ctx context.Context, now time.Time) {
370+
keys, err := i.imageIdentityCacheKeysToRefresh(ctx, now)
371+
if err != nil {
372+
log.G(ctx).WithError(err).Debug("failed to list image identity cache refresh candidates")
373+
return
374+
}
375+
if len(keys) == 0 {
376+
return
377+
}
378+
379+
var g errgroup.Group
380+
g.SetLimit(imageIdentityRefreshConcurrency)
381+
for _, key := range keys {
382+
g.Go(func() error {
383+
if err := i.refreshImageIdentityCacheKey(ctx, key); err != nil {
384+
log.G(ctx).WithError(err).WithField("cacheKey", key).Debug("failed to refresh image identity cache entry")
385+
}
386+
return nil
387+
})
388+
}
389+
_ = g.Wait()
390+
}
391+
392+
func (i *ImageService) pruneImageIdentityInMemoryCacheEntries(now time.Time) {
393+
i.identity.cacheMu.Lock()
394+
pruneImageIdentityCacheEntries(i.identity.cache, now)
395+
i.identity.cacheMu.Unlock()
396+
}
397+
398+
func (i *ImageService) pruneImageIdentityCacheStore(ctx context.Context, now time.Time) {
399+
if i.identity.cacheStore == nil {
400+
return
401+
}
402+
if err := i.identity.cacheStore.PruneExpired(ctx, now); err != nil {
403+
log.G(ctx).WithError(err).Debug("failed to prune expired image identity cache entries")
404+
}
405+
}
406+
407+
func (i *ImageService) runImageIdentityCacheMaintenance(ctx context.Context, now time.Time) {
408+
i.refreshImageIdentityCache(ctx, now)
409+
i.pruneImageIdentityInMemoryCacheEntries(now)
410+
i.pruneImageIdentityCacheStore(ctx, now)
411+
}
412+
413+
func (i *ImageService) refreshImageIdentityCacheKey(ctx context.Context, cacheKey string) error {
414+
imageDigest, bestDigest, bestPlatform, err := parseImageIdentityCacheKey(cacheKey)
415+
if err != nil {
416+
return err
417+
}
418+
419+
imgs, err := i.images.List(ctx, "target.digest=="+imageDigest.String())
420+
if err != nil {
421+
return err
422+
}
423+
if len(imgs) == 0 {
424+
return nil
425+
}
426+
427+
platformMatcher, err := imageIdentityPlatformMatcher(bestPlatform)
428+
if err != nil {
429+
return err
430+
}
431+
432+
for _, img := range imgs {
433+
multi, err := i.multiPlatformSummary(ctx, img, platformMatcher)
434+
if err != nil {
435+
continue
436+
}
437+
if multi.Best == nil {
438+
continue
439+
}
440+
if bestDigest != "" && multi.Best.Target().Digest != bestDigest {
441+
continue
442+
}
443+
computedSignature, hasTransientVerificationError := i.computeSignatureIdentity(ctx, img.Target, multi)
444+
_ = i.cacheComputedSignatureIdentity(ctx, cacheKey, img.Target, computedSignature, hasTransientVerificationError)
445+
return nil
446+
}
447+
448+
return nil
449+
}
450+
451+
func imageIdentityPlatformMatcher(platform string) (platforms.MatchComparer, error) {
452+
if platform == "" {
453+
return matchAnyWithPreference(platforms.Default(), nil), nil
454+
}
455+
parsed, err := platforms.Parse(platform)
456+
if err != nil {
457+
return nil, err
458+
}
459+
return platforms.Only(parsed), nil
460+
}
461+
462+
func parseImageIdentityCacheKey(cacheKey string) (digest.Digest, digest.Digest, string, error) {
463+
parts := strings.Split(cacheKey, "|")
464+
if len(parts) != 3 {
465+
return "", "", "", errors.Errorf("invalid image identity cache key %q", cacheKey)
466+
}
467+
468+
imageDigest, err := digest.Parse(parts[0])
469+
if err != nil {
470+
return "", "", "", errors.Wrapf(err, "invalid image digest in image identity cache key %q", cacheKey)
471+
}
472+
473+
var bestDigest digest.Digest
474+
if parts[1] != "" {
475+
bestDigest, err = digest.Parse(parts[1])
476+
if err != nil {
477+
return "", "", "", errors.Wrapf(err, "invalid best digest in image identity cache key %q", cacheKey)
478+
}
479+
}
480+
481+
return imageDigest, bestDigest, parts[2], nil
482+
}
483+
484+
func (i *ImageService) startImageIdentityCacheRefresh() {
485+
i.identity.cacheMu.Lock()
486+
if i.identity.stopRefresh != nil {
487+
i.identity.cacheMu.Unlock()
488+
return
489+
}
490+
i.identity.stopRefresh = make(chan struct{})
491+
i.identity.refreshDone = make(chan struct{})
492+
stopCh := i.identity.stopRefresh
493+
doneCh := i.identity.refreshDone
494+
i.identity.cacheMu.Unlock()
495+
496+
go func() {
497+
defer close(doneCh)
498+
runMaintenance := func() {
499+
now := time.Now()
500+
501+
maintenanceCtx, cancel := context.WithTimeout(context.Background(), imageIdentityRefreshTimeout)
502+
defer cancel()
503+
i.runImageIdentityCacheMaintenance(maintenanceCtx, now)
504+
}
505+
506+
// Run one pass right after startup to refresh expired entries and prune stale ones.
507+
runMaintenance()
508+
509+
timer := time.NewTicker(imageIdentityRefreshInterval)
510+
defer timer.Stop()
511+
for {
512+
select {
513+
case <-stopCh:
514+
return
515+
case <-timer.C:
516+
runMaintenance()
517+
}
518+
}
519+
}()
520+
}
521+
522+
func (i *ImageService) stopImageIdentityCacheRefresh() {
523+
i.identity.stopRefreshOnce.Do(func() {
524+
i.identity.cacheMu.Lock()
525+
stopCh := i.identity.stopRefresh
526+
doneCh := i.identity.refreshDone
527+
i.identity.cacheMu.Unlock()
528+
if stopCh != nil {
529+
close(stopCh)
530+
}
531+
if doneCh != nil {
532+
<-doneCh
533+
}
534+
})
535+
}
536+
317537
func (i *ImageService) warmImageIdentityCache(ctx context.Context, img c8dimages.Image) {
318538
if i.policyVerifier == nil {
319539
return
Collapse file

‎daemon/containerd/image_identity_test.go‎

Copy file name to clipboardExpand all lines: daemon/containerd/image_identity_test.go
+75Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,11 @@ import (
1717
"github.com/containerd/containerd/v2/pkg/labels"
1818
"github.com/containerd/containerd/v2/pkg/namespaces"
1919
"github.com/containerd/containerd/v2/plugins/content/local"
20+
"github.com/containerd/platforms"
2021
imagetypes "github.com/moby/moby/api/types/image"
2122
"github.com/moby/moby/v2/daemon/containerd/identitycache"
2223
"github.com/moby/moby/v2/daemon/internal/builder-next/exporter"
24+
"github.com/moby/moby/v2/internal/testutil/specialimage"
2325
"github.com/opencontainers/go-digest"
2426
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
2527
bolt "go.etcd.io/bbolt"
@@ -208,6 +210,79 @@ func TestImageIdentityCachePersistsAcrossRestart(t *testing.T) {
208210
assert.Check(t, is.Equal(res.Name, "persisted"))
209211
}
210212

213+
func TestImageIdentityCacheKeysToRefresh(t *testing.T) {
214+
ctx := namespaces.WithNamespace(t.Context(), "testing-"+t.Name())
215+
now := time.Now().UTC()
216+
217+
service := &ImageService{
218+
identity: imageIdentityState{
219+
cache: map[string]imageIdentityCacheEntry{
220+
"expired": {
221+
CachedAt: now.Add(-2 * time.Hour),
222+
ExpiresAt: now.Add(-time.Minute),
223+
},
224+
"near": {
225+
CachedAt: now.Add(-time.Hour),
226+
ExpiresAt: now.Add(imageIdentityRefreshAhead / 2),
227+
},
228+
"fresh": {
229+
CachedAt: now.Add(-time.Minute),
230+
ExpiresAt: now.Add(imageIdentityRefreshAhead + time.Hour),
231+
},
232+
},
233+
},
234+
}
235+
236+
keys, err := service.imageIdentityCacheKeysToRefresh(ctx, now)
237+
assert.NilError(t, err)
238+
assert.Check(t, is.DeepEqual(keys, []string{"expired", "near"}))
239+
}
240+
241+
func TestRefreshImageIdentityCacheUpdatesExpiredPersistedEntry(t *testing.T) {
242+
ctx := namespaces.WithNamespace(t.Context(), "testing-"+t.Name())
243+
244+
blobsDir := t.TempDir()
245+
idx, err := specialimage.MultiLayer(blobsDir)
246+
assert.NilError(t, err)
247+
248+
cs := &blobsDirContentStore{blobs: filepath.Join(blobsDir, "blobs/sha256")}
249+
service := fakeImageService(t, ctx, cs)
250+
251+
img, err := service.images.Create(ctx, imagesFromIndex(idx)[0])
252+
assert.NilError(t, err)
253+
254+
backend, err := identitycache.NewBoltDBBackend(t.TempDir())
255+
assert.NilError(t, err)
256+
defer backend.Close()
257+
258+
service.identity.cacheStore = backend
259+
service.identity.cache = map[string]imageIdentityCacheEntry{}
260+
261+
multi, err := service.multiPlatformSummary(ctx, img, matchAnyWithPreference(platforms.Default(), nil))
262+
assert.NilError(t, err)
263+
if multi.Best == nil {
264+
t.Fatal("expected best manifest for test image")
265+
}
266+
267+
key := imageIdentityCacheKey(img.Target.Digest.String(), multi.Best.Target().Digest.String(), platforms.FormatAll(multi.BestPlatform))
268+
now := time.Now().UTC()
269+
assert.NilError(t, backend.Store(ctx, key, imageIdentityCacheEntry{
270+
CachedAt: now.Add(-2 * time.Hour),
271+
ExpiresAt: now.Add(-time.Minute),
272+
}, now))
273+
274+
_, ok, err := backend.Load(ctx, key, now)
275+
assert.NilError(t, err)
276+
assert.Check(t, !ok)
277+
278+
service.refreshImageIdentityCache(ctx, now)
279+
280+
refreshed, ok, err := backend.Load(ctx, key, now)
281+
assert.NilError(t, err)
282+
assert.Check(t, ok)
283+
assert.Check(t, refreshed.ExpiresAt.After(now))
284+
}
285+
211286
type timeoutNetError struct{}
212287

213288
func (timeoutNetError) Error() string { return "timeout network error" }

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.