From bc8fea3d478f0a0eb841255c966ff723724e9da3 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 30 Sep 2021 17:02:48 +0900 Subject: [PATCH 1/3] build: support --output=(oci|tar|docker|image) Signed-off-by: Akihiro Suda --- README.md | 9 ++++++- cmd/nerdctl/build.go | 57 +++++++++++++++++++------------------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 85de76a363d..616887dfe3c 100644 --- a/README.md +++ b/README.md @@ -593,11 +593,18 @@ Flags: - :whale: `--target`: Set the target build stage to build - :whale: `--build-arg`: Set build-time variables - :whale: `--no-cache`: Do not use cache when building the image +- :whale: `--output=OUTPUT`: Output destination (format: type=local,dest=path) + - :whale: `type=local,dest=path/to/output-dir`: Local directory + - :whale: `type=oci[,dest=path/to/output.tar]`: Docker/OCI dual-format tar ball (compatible with `docker buildx build`) + - :whale: `type=docker[,dest=path/to/output.tar]`: Docker format tar ball (compatible with `docker buildx build`) + - :whale: `type=tar[,dest=path/to/output.tar]`: Raw tar ball + - :whale: `type=image,name=example.com/image,push=true`: Push to a registry (see [`buildctl build`](https://github.com/moby/buildkit/tree/v0.9.0#imageregistry) documentation) - :whale: `--progress=(auto|plain|tty)`: Set type of progress output (auto, plain, tty). Use plain to show container output - :whale: `--secret`: Secret file to expose to the build: id=mysecret,src=/local/secret - :whale: `--ssh`: SSH agent socket or keys to expose to the build (format: `default|[=|[,]]`) +- :whale: `-q, --quiet`: Suppress the build output and print image ID on success -Unimplemented `docker build` flags: `--add-host`, `--cache-from`, `--iidfile`, `--label`, `--network`, `--platform`, `--quiet`, `--squash` +Unimplemented `docker build` flags: `--add-host`, `--cache-from`, `--iidfile`, `--label`, `--network`, `--platform`, `--squash` ### :whale: nerdctl commit Create a new image from a container's changes diff --git a/cmd/nerdctl/build.go b/cmd/nerdctl/build.go index 1dfa836dc99..11b9651b7a0 100644 --- a/cmd/nerdctl/build.go +++ b/cmd/nerdctl/build.go @@ -17,7 +17,7 @@ package main import ( - "fmt" + "io" "os" "os/exec" "strings" @@ -69,7 +69,7 @@ var buildCommand = &cli.Command{ &cli.StringFlag{ Name: "output", Aliases: []string{"o"}, - Value: "type=docker", + Usage: "Output destination (format: type=local,dest=path)", }, &cli.StringFlag{ Name: "progress", @@ -93,7 +93,7 @@ func buildAction(clicontext *cli.Context) error { return err } - buildctlBinary, buildctlArgs, err := generateBuildctlArgs(clicontext) + buildctlBinary, buildctlArgs, needsLoading, err := generateBuildctlArgs(clicontext) if err != nil { return err } @@ -102,9 +102,14 @@ func buildAction(clicontext *cli.Context) error { buildctlCmd := exec.Command(buildctlBinary, buildctlArgs...) buildctlCmd.Env = os.Environ() - buildctlStdout, err := buildctlCmd.StdoutPipe() - if err != nil { - return err + var buildctlStdout io.Reader + if needsLoading { + buildctlStdout, err = buildctlCmd.StdoutPipe() + if err != nil { + return err + } + } else { + buildctlCmd.Stdout = clicontext.App.Writer } buildctlCmd.Stderr = clicontext.App.ErrWriter @@ -112,11 +117,7 @@ func buildAction(clicontext *cli.Context) error { return err } - localBuild, err := isLocalBuild(clicontext) - if err != nil { - return err - } - if !localBuild { + if needsLoading { if err = loadImage(buildctlStdout, clicontext); err != nil { return err } @@ -129,24 +130,29 @@ func buildAction(clicontext *cli.Context) error { return nil } -func generateBuildctlArgs(clicontext *cli.Context) (string, []string, error) { +func generateBuildctlArgs(clicontext *cli.Context) (string, []string, bool, error) { + var needsLoading bool if clicontext.NArg() < 1 { - return "", nil, errors.New("context needs to be specified") + return "", nil, false, errors.New("context needs to be specified") } buildContext := clicontext.Args().First() if buildContext == "-" || strings.Contains(buildContext, "://") { - return "", nil, errors.Errorf("unsupported build context: %q", buildContext) + return "", nil, false, errors.Errorf("unsupported build context: %q", buildContext) } buildctlBinary, err := buildkitutil.BuildctlBinary() if err != nil { - return "", nil, err + return "", nil, false, err } - output := fmt.Sprintf("--output=%s", clicontext.String("output")) + output := clicontext.String("output") + if output == "" { + output = "type=docker" + needsLoading = true + } if tagSlice := strutil.DedupeStrSlice(clicontext.StringSlice("tag")); len(tagSlice) > 0 { if len(tagSlice) > 1 { - return "", nil, errors.Errorf("specifying multiple -t is not supported yet") + return "", nil, false, errors.Errorf("specifying multiple -t is not supported yet") } output += ",name=" + tagSlice[0] } @@ -159,7 +165,7 @@ func generateBuildctlArgs(clicontext *cli.Context) (string, []string, error) { "--frontend=dockerfile.v0", "--local=context=" + buildContext, "--local=dockerfile=" + buildContext, - output, + "--output=" + output, }...) if filename := clicontext.String("file"); filename != "" { @@ -190,18 +196,5 @@ func generateBuildctlArgs(clicontext *cli.Context) (string, []string, error) { buildctlArgs = append(buildctlArgs, "--ssh="+s) } - return buildctlBinary, buildctlArgs, nil -} - -func isLocalBuild(clicontext *cli.Context) (bool, error) { - opts, err := strutil.ParseCSVMap(clicontext.String("output")) - if err != nil { - return false, err - } - if v, ok := opts["type"]; ok { - if strings.TrimSpace(strings.ToLower(v)) == "local" { - return true, nil - } - } - return false, nil + return buildctlBinary, buildctlArgs, needsLoading, nil } From 77d602ea07ab786446fd1014fca20e84cbbdebda Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 30 Sep 2021 17:21:10 +0900 Subject: [PATCH 2/3] build: support --quiet Signed-off-by: Akihiro Suda --- cmd/nerdctl/build.go | 14 ++++++++++++-- cmd/nerdctl/load.go | 14 ++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cmd/nerdctl/build.go b/cmd/nerdctl/build.go index 11b9651b7a0..a0afd0a0482 100644 --- a/cmd/nerdctl/build.go +++ b/cmd/nerdctl/build.go @@ -84,6 +84,11 @@ var buildCommand = &cli.Command{ Name: "ssh", Usage: "SSH agent socket or keys to expose to the build (format: default|[=|[,]])", }, + &cli.BoolFlag{ + Name: "quiet", + Aliases: []string{"q"}, + Usage: "Suppress the build output and print image ID on success", + }, }, } @@ -98,6 +103,8 @@ func buildAction(clicontext *cli.Context) error { return err } + quiet := clicontext.Bool("quiet") + logrus.Debugf("running %s %v", buildctlBinary, buildctlArgs) buildctlCmd := exec.Command(buildctlBinary, buildctlArgs...) buildctlCmd.Env = os.Environ() @@ -111,14 +118,17 @@ func buildAction(clicontext *cli.Context) error { } else { buildctlCmd.Stdout = clicontext.App.Writer } - buildctlCmd.Stderr = clicontext.App.ErrWriter + + if !quiet { + buildctlCmd.Stderr = clicontext.App.ErrWriter + } if err := buildctlCmd.Start(); err != nil { return err } if needsLoading { - if err = loadImage(buildctlStdout, clicontext); err != nil { + if err = loadImage(buildctlStdout, clicontext, quiet); err != nil { return err } } diff --git a/cmd/nerdctl/load.go b/cmd/nerdctl/load.go index 3ca4c1998dc..dbe0d37af1d 100644 --- a/cmd/nerdctl/load.go +++ b/cmd/nerdctl/load.go @@ -55,10 +55,10 @@ func loadAction(clicontext *cli.Context) error { defer f.Close() in = f } - return loadImage(in, clicontext) + return loadImage(in, clicontext, false) } -func loadImage(in io.Reader, clicontext *cli.Context) error { +func loadImage(in io.Reader, clicontext *cli.Context, quiet bool) error { client, ctx, cancel, err := newClient(clicontext, containerd.WithDefaultPlatform(platforms.DefaultStrict())) if err != nil { return err @@ -74,12 +74,18 @@ func loadImage(in io.Reader, clicontext *cli.Context) error { image := containerd.NewImage(client, img) // TODO: Show unpack status - fmt.Fprintf(clicontext.App.Writer, "unpacking %s (%s)...", img.Name, img.Target.Digest) + if !quiet { + fmt.Fprintf(clicontext.App.Writer, "unpacking %s (%s)...", img.Name, img.Target.Digest) + } err = image.Unpack(ctx, sn) if err != nil { return err } - fmt.Fprintf(clicontext.App.Writer, "done\n") + if quiet { + fmt.Fprintln(clicontext.App.Writer, img.Target.Digest) + } else { + fmt.Fprintf(clicontext.App.Writer, "done\n") + } } return nil From 6f68f8945d8df09be70600e96ee050b74d7d4d23 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 30 Sep 2021 17:35:24 +0900 Subject: [PATCH 3/3] build: support --cache-from and --cache-to Signed-off-by: Akihiro Suda --- README.md | 4 +++- cmd/nerdctl/build.go | 37 +++++++++++++++++++++++++++++ pkg/composer/serviceparser/build.go | 6 ++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 616887dfe3c..74857b56ebe 100644 --- a/README.md +++ b/README.md @@ -603,8 +603,10 @@ Flags: - :whale: `--secret`: Secret file to expose to the build: id=mysecret,src=/local/secret - :whale: `--ssh`: SSH agent socket or keys to expose to the build (format: `default|[=|[,]]`) - :whale: `-q, --quiet`: Suppress the build output and print image ID on success +- :whale: `--cache-from=CACHE`: External cache sources (eg. user/app:cache, type=local,src=path/to/dir) (compatible with `docker buildx build`) +- :whale: `--cache-to=CACHE`: Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir) (compatible with `docker buildx build`) -Unimplemented `docker build` flags: `--add-host`, `--cache-from`, `--iidfile`, `--label`, `--network`, `--platform`, `--squash` +Unimplemented `docker build` flags: `--add-host`, `--iidfile`, `--label`, `--network`, `--platform`, `--squash` ### :whale: nerdctl commit Create a new image from a container's changes diff --git a/cmd/nerdctl/build.go b/cmd/nerdctl/build.go index a0afd0a0482..928ede461b3 100644 --- a/cmd/nerdctl/build.go +++ b/cmd/nerdctl/build.go @@ -20,6 +20,7 @@ import ( "io" "os" "os/exec" + "strconv" "strings" "path/filepath" @@ -89,6 +90,14 @@ var buildCommand = &cli.Command{ Aliases: []string{"q"}, Usage: "Suppress the build output and print image ID on success", }, + &cli.StringSliceFlag{ + Name: "cache-from", + Usage: "External cache sources (eg. user/app:cache, type=local,src=path/to/dir)", + }, + &cli.StringSliceFlag{ + Name: "cache-to", + Usage: "Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir)", + }, }, } @@ -192,6 +201,20 @@ func generateBuildctlArgs(clicontext *cli.Context) (string, []string, bool, erro for _, ba := range strutil.DedupeStrSlice(clicontext.StringSlice("build-arg")) { buildctlArgs = append(buildctlArgs, "--opt=build-arg:"+ba) + + // Support `--build-arg BUILDKIT_INLINE_CACHE=1` for compatibility with `docker buildx build` + // https://github.com/docker/buildx/blob/v0.6.3/docs/reference/buildx_build.md#-export-build-cache-to-an-external-cache-destination---cache-to + if strings.HasPrefix(ba, "BUILDKIT_INLINE_CACHE=") { + bic := strings.TrimPrefix(ba, "BUILDKIT_INLINE_CACHE=") + bicParsed, err := strconv.ParseBool(bic) + if err == nil { + if bicParsed { + buildctlArgs = append(buildctlArgs, "--export-cache=type=inline") + } + } else { + logrus.WithError(err).Warnf("invalid BUILDKIT_INLINE_CACHE: %q", bic) + } + } } if clicontext.Bool("no-cache") { @@ -206,5 +229,19 @@ func generateBuildctlArgs(clicontext *cli.Context) (string, []string, bool, erro buildctlArgs = append(buildctlArgs, "--ssh="+s) } + for _, s := range strutil.DedupeStrSlice(clicontext.StringSlice("cache-from")) { + if !strings.Contains(s, "type=") { + s = "type=registry,ref=" + s + } + buildctlArgs = append(buildctlArgs, "--import-cache="+s) + } + + for _, s := range strutil.DedupeStrSlice(clicontext.StringSlice("cache-to")) { + if !strings.Contains(s, "type=") { + s = "type=registry,ref=" + s + } + buildctlArgs = append(buildctlArgs, "--export-cache="+s) + } + return buildctlBinary, buildctlArgs, needsLoading, nil } diff --git a/pkg/composer/serviceparser/build.go b/pkg/composer/serviceparser/build.go index 87e6dfe7d27..e9cebce216d 100644 --- a/pkg/composer/serviceparser/build.go +++ b/pkg/composer/serviceparser/build.go @@ -29,7 +29,7 @@ import ( func parseBuildConfig(c *types.BuildConfig, project *types.Project, imageName string) (*Build, error) { if unknown := reflectutil.UnknownNonEmptyFields(c, - "Context", "Dockerfile", "Args", "Target", + "Context", "Dockerfile", "Args", "CacheFrom", "Target", ); len(unknown) > 0 { logrus.Warnf("Ignoring: build: %+v", unknown) } @@ -66,6 +66,10 @@ func parseBuildConfig(c *types.BuildConfig, project *types.Project, imageName st } } + for _, s := range c.CacheFrom { + b.BuildArgs = append(b.BuildArgs, "--cache-from="+s) + } + if c.Target != "" { b.BuildArgs = append(b.BuildArgs, "--target="+c.Target) }