Skip to content

Commit

Permalink
Add image squash ut & Update docs/command-reference.md
Browse files Browse the repository at this point in the history
Signed-off-by: weipeng <weip3@chinatelecom.cn>
  • Loading branch information
weipeng committed Dec 31, 2024
1 parent 313960f commit 0282a02
Show file tree
Hide file tree
Showing 5 changed files with 166 additions and 26 deletions.
1 change: 1 addition & 0 deletions cmd/nerdctl/image/image_squash.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ func addSquashFlags(cmd *cobra.Command) {
cmd.Flags().StringP("message", "m", "", "Commit message")
}

// NewSquashCommand returns a new `squash` command to compress the number of layers of the image
func NewSquashCommand() *cobra.Command {
var squashCommand = &cobra.Command{
Use: "squash [flags] SOURCE_IMAGE TAG_IMAGE",
Expand Down
95 changes: 95 additions & 0 deletions cmd/nerdctl/image/image_squash_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package image

import (
"fmt"
"testing"

"gotest.tools/v3/assert"

"github.com/containerd/nerdctl/v2/pkg/testutil"
"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
"github.com/containerd/nerdctl/v2/pkg/testutil/test"
)

func squashIdentifierName(identifier string) string {
return fmt.Sprintf("%s-squash", identifier)
}

func secondCommitedIdentifierName(identifier string) string {
return fmt.Sprintf("%s-second", identifier)
}

func TestSquash(t *testing.T) {
testCase := nerdtest.Setup()

testCase.SubTests = []*test.Case{
{
Description: "by layer count",
Require: nerdtest.CGroup,
NoParallel: true,
Cleanup: func(data test.Data, helpers test.Helpers) {
identifier := data.Identifier()
secondIdentifier := secondCommitedIdentifierName(identifier)
squashIdentifier := squashIdentifierName(identifier)
helpers.Anyhow("rm", "-f", identifier)
helpers.Anyhow("rm", "-f", secondIdentifier)
helpers.Anyhow("rm", "-f", squashIdentifier)

helpers.Anyhow("rmi", "-f", secondIdentifier)
helpers.Anyhow("rmi", "-f", identifier)
helpers.Anyhow("rmi", "-f", squashIdentifier)
helpers.Anyhow("image", "prune", "-f")
},
Setup: func(data test.Data, helpers test.Helpers) {
identifier := data.Identifier()
helpers.Ensure("run", "-d", "--name", identifier, testutil.CommonImage, "sleep", nerdtest.Infinity)
helpers.Ensure("exec", identifier, "sh", "-euxc", `echo hello-first-commit > /foo`)
helpers.Ensure("commit", "-c", `CMD ["cat", "/foo"]`, "-m", `first commit`, "--pause=true", identifier, identifier)
out := helpers.Capture("run", "--rm", identifier)
assert.Equal(t, out, "hello-first-commit\n")

secondIdentifier := secondCommitedIdentifierName(identifier)
helpers.Ensure("run", "-d", "--name", secondIdentifier, identifier, "sleep", nerdtest.Infinity)
helpers.Ensure("exec", secondIdentifier, "sh", "-euxc", `echo hello-second-commit > /bar && echo hello-squash-commit > /foo`)
helpers.Ensure("commit", "-c", `CMD ["cat", "/foo", "/bar"]`, "-m", `second commit`, "--pause=true", secondIdentifier, secondIdentifier)
out = helpers.Capture("run", "--rm", secondIdentifier)
assert.Equal(t, out, "hello-squash-commit\nhello-second-commit\n")

squashIdentifier := squashIdentifierName(identifier)
helpers.Ensure("image", "squash", "-c=2", "-m", "squash commit", secondIdentifier, squashIdentifier)
out = helpers.Capture("run", "--rm", squashIdentifier)
assert.Equal(t, out, "hello-squash-commit\nhello-second-commit\n")
},
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
identifier := data.Identifier()

squashIdentifier := squashIdentifierName(identifier)
return helpers.Command("image", "history", "--human=true", "--format=json", squashIdentifier)
},
Expected: test.Expects(0, nil, func(stdout string, info string, t *testing.T) {
history, err := decode(stdout)
assert.NilError(t, err, info)
assert.Equal(t, len(history), 3, info)
assert.Equal(t, history[0].Comment, "squash commit", info)
}),
},
}

testCase.Run(t)
}
1 change: 0 additions & 1 deletion cmd/nerdctl/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,6 @@ Config file ($NERDCTL_TOML): %s
image.NewTagCommand(),
image.NewRmiCommand(),
image.NewHistoryCommand(),
image.NewSquashCommand(),
// #endregion

// #region System
Expand Down
19 changes: 19 additions & 0 deletions docs/command-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -973,6 +973,25 @@ Flags:
- `--platform=<PLATFORM>` : Convert content for a specific platform
- `--all-platforms` : Convert content for all platforms (default: false)

### :nerd_face: nerdctl image squash

Squash an image layers.

Usage: `nerdctl image squash [OPTIONS] SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG]`

Example:

```bash
nerdctl image pull example.com/foo:latest
nerdctl image squash --layer-count=2 --message="squash commit" example.com/foo:latest example.com/foo:squashed
```

Flags:
- `-c --layer-count=<COUNT>`: The number of layers that can be compressed
- `-d --layer-digest=<DIGEST>`: The digest of the layer to be compressed
- `-m --message=<MESSAGE>`: Commit message for the squashed image
- `-a --author=<AUTHOR>`: Author of the squashed image

## Registry

### :whale: nerdctl login
Expand Down
76 changes: 51 additions & 25 deletions pkg/cmd/image/squash.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,20 +44,23 @@ import (
"github.com/containerd/log"

"github.com/containerd/nerdctl/v2/pkg/api/types"
"github.com/containerd/nerdctl/v2/pkg/idutil/imagewalker"
"github.com/containerd/nerdctl/v2/pkg/imgutil"
)

const (
emptyDigest = digest.Digest("")
)

// squashImage is the image for squash operation
type squashImage struct {
ClientImage containerd.Image
Config ocispec.Image
Image images.Image
Manifest *ocispec.Manifest
clientImage containerd.Image
config ocispec.Image
image images.Image
manifest *ocispec.Manifest
}

// squashRuntime is the runtime for squash operation
type squashRuntime struct {
opt types.ImageSquashOptions

Expand All @@ -70,6 +73,7 @@ type squashRuntime struct {
snapshotter snapshots.Snapshotter
}

// initImage initializes the squashImage based on the source image reference
func (sr *squashRuntime) initImage(ctx context.Context) (*squashImage, error) {
containerImage, err := sr.imageStore.Get(ctx, sr.opt.SourceImageRef)
if err != nil {
Expand All @@ -86,20 +90,21 @@ func (sr *squashRuntime) initImage(ctx context.Context) (*squashImage, error) {
return &squashImage{}, err
}
resImage := &squashImage{
ClientImage: clientImage,
Config: config,
Image: containerImage,
Manifest: manifest,
clientImage: clientImage,
config: config,
image: containerImage,
manifest: manifest,
}
return resImage, err
}

// generateSquashLayer generates the squash layer based on the given options
func (sr *squashRuntime) generateSquashLayer(image *squashImage) ([]ocispec.Descriptor, error) {
// get the layer descriptors by the layer digest
if sr.opt.SquashLayerDigest != "" {
find := false
var res []ocispec.Descriptor
for _, layer := range image.Manifest.Layers {
for _, layer := range image.manifest.Layers {
if layer.Digest.String() == sr.opt.SquashLayerDigest {
find = true
}
Expand All @@ -114,13 +119,14 @@ func (sr *squashRuntime) generateSquashLayer(image *squashImage) ([]ocispec.Desc
}

// get the layer descriptors by the layer count
if sr.opt.SquashLayerCount > 1 && sr.opt.SquashLayerCount <= len(image.Manifest.Layers) {
return image.Manifest.Layers[len(image.Manifest.Layers)-sr.opt.SquashLayerCount:], nil
if sr.opt.SquashLayerCount > 1 && sr.opt.SquashLayerCount <= len(image.manifest.Layers) {
return image.manifest.Layers[len(image.manifest.Layers)-sr.opt.SquashLayerCount:], nil
}

return nil, fmt.Errorf("invalid squash option: %w", errdefs.ErrInvalidArgument)
}

// applyLayersToSnapshot applies the layers to the snapshot
func (sr *squashRuntime) applyLayersToSnapshot(ctx context.Context, mount []mount.Mount, layers []ocispec.Descriptor) error {
for _, layer := range layers {
if _, err := sr.differ.Apply(ctx, layer, mount); err != nil {
Expand Down Expand Up @@ -157,7 +163,7 @@ func (sr *squashRuntime) createDiff(ctx context.Context, snapshotName string) (o

func (sr *squashRuntime) generateBaseImageConfig(ctx context.Context, image *squashImage, remainingLayerCount int) (ocispec.Image, error) {
// generate squash squashImage config
orginalConfig, _, err := imgutil.ReadImageConfig(ctx, image.ClientImage) // aware of img.platform
orginalConfig, _, err := imgutil.ReadImageConfig(ctx, image.clientImage) // aware of img.platform
if err != nil {
return ocispec.Image{}, err
}
Expand Down Expand Up @@ -257,9 +263,9 @@ func (sr *squashRuntime) writeContentsForImage(ctx context.Context, snName strin
return newMfstDesc, configDesc.Digest, nil
}

// createSquashImage creates a new squashImage in the image store.
func (sr *squashRuntime) createSquashImage(ctx context.Context, img images.Image) (images.Image, error) {
newImg, err := sr.imageStore.Update(ctx, img)
log.G(ctx).Infof("updated new squashImage %s", img.Name)
if err != nil {
// if err is `not found` in the message then create the squashImage, otherwise return the error
if !errdefs.IsNotFound(err) {
Expand All @@ -268,13 +274,12 @@ func (sr *squashRuntime) createSquashImage(ctx context.Context, img images.Image
if _, err := sr.imageStore.Create(ctx, img); err != nil {
return newImg, fmt.Errorf("failed to create new squashImage %s: %w", img.Name, err)
}
log.G(ctx).Infof("created new squashImage %s", img.Name)
}
return newImg, nil
}

// generateCommitImageConfig returns commit oci image config based on the container's image.
func (sr *squashRuntime) generateCommitImageConfig(ctx context.Context, baseConfig ocispec.Image, diffID digest.Digest) (ocispec.Image, error) {
func (sr *squashRuntime) generateCommitImageConfig(ctx context.Context, baseImg images.Image, baseConfig ocispec.Image, diffID digest.Digest) (ocispec.Image, error) {
createdTime := time.Now()
arch := baseConfig.Architecture
if arch == "" {
Expand All @@ -292,6 +297,7 @@ func (sr *squashRuntime) generateCommitImageConfig(ctx context.Context, baseConf
}
comment := strings.TrimSpace(sr.opt.Message)

baseImageDigest := strings.Split(baseImg.Target.Digest.String(), ":")[1][:12]
return ocispec.Image{
Platform: ocispec.Platform{
Architecture: arch,
Expand All @@ -307,7 +313,7 @@ func (sr *squashRuntime) generateCommitImageConfig(ctx context.Context, baseConf
},
History: append(baseConfig.History, ocispec.History{
Created: &createdTime,
CreatedBy: "",
CreatedBy: fmt.Sprintf("squash from %s", baseImageDigest),
Author: author,
Comment: comment,
EmptyLayer: false,
Expand All @@ -317,19 +323,38 @@ func (sr *squashRuntime) generateCommitImageConfig(ctx context.Context, baseConf

// Squash will squash the image with the given options.
func Squash(ctx context.Context, client *containerd.Client, option types.ImageSquashOptions) error {
var srcName string
walker := &imagewalker.ImageWalker{
Client: client,
OnFound: func(ctx context.Context, found imagewalker.Found) error {
if srcName == "" {
srcName = found.Image.Name
}
return nil
},
}
matchCount, err := walker.Walk(ctx, option.SourceImageRef)
if err != nil {
return err
}
if matchCount < 1 {
return fmt.Errorf("%s: not found", option.SourceImageRef)
}

option.SourceImageRef = srcName
sr := newSquashRuntime(client, option)
ctx = namespaces.WithNamespace(ctx, sr.namespace)
// init squashImage
image, err := sr.initImage(ctx)
img, err := sr.initImage(ctx)
if err != nil {
return err
}
// generate squash layers
sLayers, err := sr.generateSquashLayer(image)
sLayers, err := sr.generateSquashLayer(img)
if err != nil {
return err
}
remainingLayerCount := len(image.Manifest.Layers) - len(sLayers)
remainingLayerCount := len(img.manifest.Layers) - len(sLayers)
// Don't gc me and clean the dirty data after 1 hour!
ctx, done, err := sr.client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(1*time.Hour))
if err != nil {
Expand All @@ -338,7 +363,7 @@ func Squash(ctx context.Context, client *containerd.Client, option types.ImageSq
defer done(ctx)

// generate remaining base squashImage config
baseImage, err := sr.generateBaseImageConfig(ctx, image, remainingLayerCount)
baseImage, err := sr.generateBaseImageConfig(ctx, img, remainingLayerCount)
if err != nil {
return err
}
Expand All @@ -348,27 +373,27 @@ func Squash(ctx context.Context, client *containerd.Client, option types.ImageSq
return err
}
// generate commit image config
imageConfig, err := sr.generateCommitImageConfig(ctx, baseImage, diffID)
imageConfig, err := sr.generateCommitImageConfig(ctx, img.image, baseImage, diffID)
if err != nil {
log.G(ctx).WithError(err).Error("failed to generate commit image config")
return fmt.Errorf("failed to generate commit image config: %w", err)
}
commitManifestDesc, _, err := sr.writeContentsForImage(ctx, sr.opt.GOptions.Snapshotter, imageConfig, image.Manifest.Layers[:remainingLayerCount], diffLayerDesc)
commitManifestDesc, _, err := sr.writeContentsForImage(ctx, sr.opt.GOptions.Snapshotter, imageConfig, img.manifest.Layers[:remainingLayerCount], diffLayerDesc)
if err != nil {
log.G(ctx).WithError(err).Error("failed to write contents for image")
return err
}
nimg := images.Image{
nImg := images.Image{
Name: sr.opt.TargetImageName,
Target: commitManifestDesc,
UpdatedAt: time.Now(),
}
_, err = sr.createSquashImage(ctx, nimg)
_, err = sr.createSquashImage(ctx, nImg)
if err != nil {
log.G(ctx).WithError(err).Error("failed to create squash image")
return err
}
cimg := containerd.NewImage(sr.client, nimg)
cimg := containerd.NewImage(sr.client, nImg)
if err := cimg.Unpack(ctx, sr.opt.GOptions.Snapshotter, containerd.WithSnapshotterPlatformCheck()); err != nil {
log.G(ctx).WithError(err).Error("failed to unpack squash image")
return err
Expand Down Expand Up @@ -434,6 +459,7 @@ func newSquashRuntime(client *containerd.Client, option types.ImageSquashOptions
}

// copied from github.com/containerd/containerd/rootfs/apply.go
// which commit hash is 597d0d76ae03e945996ae6e003dae0c668fa158e by McGowan
func uniquePart() string {
t := time.Now()
var b [3]byte
Expand Down

0 comments on commit 0282a02

Please sign in to comment.