diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..5f7bd918a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,20 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "args": [ + "--image-name", + "us.gcr.io/kubernetes-dev/infra-build", + "--dry-run" + ] + } + ] +} diff --git a/actions/build-push-to-dockerhub/README.md b/actions/build-push-to-dockerhub/README.md index 4827dbb89..50a369c3d 100644 --- a/actions/build-push-to-dockerhub/README.md +++ b/actions/build-push-to-dockerhub/README.md @@ -1,6 +1,9 @@ # build-push-to-dockerhub -This is a composite GitHub Action, used to build Docker images and push them to DockerHub. +This is a composite GitHub Action, used to build Docker images and push them to +DockerHub. A SBOM and provenenace attestation will be generated too, and pushed +to the registry and the GitHub signature store. + It uses `get-vault-secrets` action to get the DockerHub username and password from Vault. Example of how to use this action in a repository: @@ -13,6 +16,7 @@ on: permissions: contents: read id-token: write + attestations: write jobs: build: diff --git a/actions/build-push-to-dockerhub/action.yaml b/actions/build-push-to-dockerhub/action.yaml index 4205a4fd1..58edb2df0 100644 --- a/actions/build-push-to-dockerhub/action.yaml +++ b/actions/build-push-to-dockerhub/action.yaml @@ -98,15 +98,21 @@ runs: tags: ${{ inputs.tags }} - name: Build and push Docker image + id: build-push uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 with: + attests: type=provenance,mode=max context: ${{ inputs.context }} + file: ${{ inputs.file }} + labels: ${{ steps.meta.outputs.labels }} platforms: ${{ inputs.platforms }} + sbom: true push: ${{ inputs.push }} tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - file: ${{ inputs.file }} - build-args: ${{ inputs.build-args }} - target: ${{ inputs.target }} - cache-from: ${{ inputs.cache-from }} - cache-to: ${{ inputs.cache-to }} + + - name: Attest build provenance + uses: actions/attest-build-provenance@5e9cb68e95676991667494a6a4e59b8a2f13e1d0 # v1.3.3 + with: + push-to-registry: ${{ inputs.push }} + subject-digest: ${{ steps.build-push.outputs.digest }} + subject-name: index.docker.io/${{ inputs.repository }} diff --git a/actions/clean-up-container-registry/action.yml b/actions/clean-up-container-registry/action.yml new file mode 100644 index 000000000..371ca3a3b --- /dev/null +++ b/actions/clean-up-container-registry/action.yml @@ -0,0 +1,116 @@ +name: Cleanup Docker Registry + +description: Clean up Docker images in a registry. + +inputs: + image_name: + description: "The name of the Docker image to clean up, e.g., `ghcr.io/myorg/myimage`." + required: true + + tag_filter: + description: "Glob pattern to filter tags. Defaults to *." + required: false + default: "*" + + exclude_tags: + description: "Comma-separated list of tags to exclude from deletion." + required: false + default: "" + + keep_latest: + description: "Number of latest images to keep." + required: true + + dry_run: + description: "Run the action in dry-run mode (true/false)." + required: false + default: "true" + + crane_version: + description: "The version of crane to use." + required: false + default: "v0.19.2" + +runs: + using: "composite" + steps: + - name: Fetch crane + id: fetch-crane + uses: robinraju/release-downloader@c39a3b234af58f0cf85888573d361fb6fa281534 # v1.10 + with: + extract: false + fileName: go-containerregistry_Linux_x86_64.tar.gz + releaseId: ${{ inputs.crane_version }} + repository: google/go-containerregistry + + - name: Download SLSA Verifier + uses: slsa-framework/slsa-verifier/actions/installer@eb7007070baa04976cb9e25a0d8034f8db030a86 #v2.5.1 + + - name: Verify Crane SLSA provenance + shell: sh + run: | + slsa-verifier \ + verify-artifact \ + "${{ fromJSON(steps.fetch-crane.outputs.downloaded_files)[0] }}" \ + --provenance-path provenance.intoto.jsonl \ + --source-uri github.com/google/go-containerregistry \ + --source-tag "${{ inputs.crane_version }}" + + - name: Extract crane + shell: sh + run: | + tar -xzvf go-containerregistry_Linux_x86_64.tar.gz crane + chmod +x crane + mv crane /usr/local/bin/crane + + - name: List images + shell: sh + id: list_images + run: | + image_name="${{ inputs.image_name }}" + tag_filter="${{ inputs.tag_filter }}" + exclude_tags="${{ inputs.exclude_tags }}" + keep_latest="${{ inputs.keep_latest }}" + dry_run="${{ inputs.dry_run }}" + + # Convert exclude_tags to an array + IFS=',' read -r -a exclude_array <<< "$exclude_tags" + + # List all image tags using crane + tags=$(crane ls $image_name) + + # Convert tags to an array + tags_array=($tags) + + # Filter tags by the glob pattern + filtered_tags=() + for tag in "${tags_array[@]}"; do + if [[ $tag == $tag_filter ]]; then + filtered_tags+=("$tag") + fi + done + + # Exclude specified tags + for exclude in "${exclude_array[@]}"; do + filtered_tags=("${filtered_tags[@]/$exclude}") + done + + # Sort tags using version sort + sorted_tags=$(printf "%s\n" "${filtered_tags[@]}" | sort -V) + + # Determine tags to remove + tags_to_remove=($(echo "$sorted_tags" | head -n -"$keep_latest")) + + if [ "$dry_run" == "true" ]; then + echo "Dry-run mode: the following tags would be removed:" + printf "%s\n" "${tags_to_remove[@]}" + else + echo "Removing the following tags:" + printf "%s\n" "${tags_to_remove[@]}" + for tag in "${tags_to_remove[@]}"; do + crane delete $image_name:$tag + done + fi + + # Output removed tags + echo "::set-output name=removed_tags::$(IFS=,; echo "${tags_to_remove[*]}")" diff --git a/actions/clean-up-container-registry/go.mod b/actions/clean-up-container-registry/go.mod new file mode 100644 index 000000000..874537eb3 --- /dev/null +++ b/actions/clean-up-container-registry/go.mod @@ -0,0 +1,30 @@ +module github.com/grafana/shared-workflows/actions/clean-up-container-registry + +go 1.22.4 + +require ( + github.com/gobwas/glob v0.2.3 + github.com/google/go-containerregistry v0.19.2 + github.com/urfave/cli/v2 v2.27.2 + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 + golang.org/x/sync v0.7.0 +) + +require ( + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/docker/cli v24.0.0+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/docker v24.0.0+incompatible // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc3 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sirupsen/logrus v1.9.1 // indirect + github.com/vbatts/tar-split v0.11.3 // indirect + github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect + golang.org/x/sys v0.15.0 // indirect +) diff --git a/actions/clean-up-container-registry/go.sum b/actions/clean-up-container-registry/go.sum new file mode 100644 index 000000000..397cb10a2 --- /dev/null +++ b/actions/clean-up-container-registry/go.sum @@ -0,0 +1,70 @@ +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v24.0.0+incompatible h1:0+1VshNwBQzQAx9lOl+OYCTCEAD8fKs/qeXMx3O0wqM= +github.com/docker/cli v24.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v24.0.0+incompatible h1:z4bf8HvONXX9Tde5lGBMQ7yCJgNahmJumdrStZAbeY4= +github.com/docker/docker v24.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.19.2 h1:TannFKE1QSajsP6hPWb5oJNgKe1IKjHukIKDUmvsV6w= +github.com/google/go-containerregistry v0.19.2/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.1 h1:Ou41VVR3nMWWmTiEUnj0OlsgOSCUFgsPAOl6jRIcVtQ= +github.com/sirupsen/logrus v1.9.1/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= +github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= +github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= +github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= +github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/actions/clean-up-container-registry/main.go b/actions/clean-up-container-registry/main.go new file mode 100644 index 000000000..c222b75c3 --- /dev/null +++ b/actions/clean-up-container-registry/main.go @@ -0,0 +1,317 @@ +package main + +import ( + "fmt" + "log" + "os" + "slices" + "strings" + "sync" + "time" + + "github.com/gobwas/glob" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/urfave/cli/v2" + "golang.org/x/sync/errgroup" +) + +type refDate struct { + ref name.Reference + date time.Time +} + +type keepReason int + +const ( + keepReasonExcluded keepReason = iota + keepReasonLatest +) + +func (kr keepReason) String() string { + switch kr { + case keepReasonExcluded: + return "excluded by filter" + case keepReasonLatest: + return "new enough" + default: + return "Unknown" + } +} + +type keptTag struct { + refDate + keepReason +} + +type repoSearchResult struct { + tagsToKeep []keptTag + tagsToRemove []refDate +} + +type Config struct { + ImageRef string + ImageRepo name.Repository + ExcludeTags []glob.Glob + TagFilter []glob.Glob + KeepLatest int + DryRun bool +} + +func buildGlobs(filters []string) ([]glob.Glob, error) { + globs := make([]glob.Glob, 0, len(filters)) + for _, filter := range filters { + g, err := glob.Compile(filter) + if err != nil { + return nil, err + } + globs = append(globs, g) + } + return globs, nil +} + +func main() { + config := Config{} + + app := &cli.App{ + Name: "docker-registry-cleanup", + Usage: "Clean up Docker images in a registry", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "image-name", + Usage: "The name of the Docker image to clean up", + Required: true, + Destination: &config.ImageRef, + Action: func(c *cli.Context, imageName string) error { + repo, err := name.NewRepository(imageName) + if err != nil { + return err + } + + config.ImageRepo = repo + + return nil + }, + }, + &cli.StringSliceFlag{ + Name: "exclude-tags", + Usage: "Tags to exclude from deletion", + Action: func(c *cli.Context, excludeTags []string) error { + globs, err := buildGlobs(excludeTags) + if err != nil { + return err + } + + config.ExcludeTags = globs + return nil + }, + }, + &cli.StringSliceFlag{ + Name: "tag-filter", + Usage: "Glob pattern to filter tags", + Action: func(c *cli.Context, tagFilter []string) error { + globs, err := buildGlobs(tagFilter) + if err != nil { + return err + } + + config.TagFilter = globs + return nil + }, + }, + &cli.IntFlag{ + Name: "keep-latest", + Usage: "Number of latest images to keep", + Value: 6, + Destination: &config.KeepLatest, + }, + &cli.BoolFlag{ + Name: "dry-run", + Usage: "Run the action in dry-run mode", + Value: true, + Destination: &config.DryRun, + }, + }, + Action: func(_ *cli.Context) error { + return run(config) + }, + } + + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} + +func run(config Config) error { + imageName := config.ImageRef + imageRepo := config.ImageRepo + + excludeTags := config.ExcludeTags + tagFilters := config.TagFilter + keepLatest := config.KeepLatest + dryRun := config.DryRun + + remoteTags, err := listRemoteTags(imageRepo) + if err != nil { + return fmt.Errorf("failed to list tags for %s: %v", imageName, err) + } + fmt.Printf("Found %d tags for %s\n", len(remoteTags), imageName) + + tags, err := getTagsToRemove(remoteTags, tagFilters, excludeTags, keepLatest, imageRepo) + if err != nil { + return fmt.Errorf("failed to determine tags to remove: %v", err) + } + + if dryRun { + fmt.Printf("Dry run mode enabled. Would have removed the following tags:\n%s", tags) + return nil + } + + err = removeTags(tags) + if err != nil { + return err + } + + return nil +} + +func listRemoteTags(imageRepo name.Repository) ([]string, error) { + return remote.List(imageRepo, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} + +func (rs repoSearchResult) String() string { + var sb strings.Builder + sb.WriteString("Tags to remove:\n") + + for _, tag := range rs.tagsToRemove { + sb.WriteString(fmt.Sprintf(" - %s\n", tag.ref.Name())) + } + + sb.WriteString("\nTags to keep:\n") + for _, tag := range rs.tagsToKeep { + sb.WriteString(fmt.Sprintf(" - %s (%s)\n", tag.ref.Name(), tag.keepReason)) + } + + return sb.String() +} + +func removeTags(tags repoSearchResult) error { + for _, tag := range tags.tagsToRemove { + if err := remote.Delete(tag.ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)); err != nil { + return fmt.Errorf("failed to delete tag %s: %v", tag.ref.Name(), err) + } + + fmt.Printf("Deleted tag %s...\n", tag.ref.Name()) + } + return nil +} + +func getTagsToRemove(allTags []string, tagFilters []glob.Glob, excludes []glob.Glob, keepLatest int, imageRepo name.Repository) (repoSearchResult, error) { + var ( + tagsToConsiderRemoving []refDate + tagsKept []keptTag + mu sync.Mutex + ) + + eg := errgroup.Group{} + eg.SetLimit(100) + + for _, tag := range allTags { + eg.Go(func() error { + fmt.Printf("Processing tag %s...\n", tag) + ref, err := name.ParseReference(fmt.Sprintf("%s:%s", imageRepo, tag)) + if err != nil { + return err + } + + if !matchesFilters(tag, tagFilters, true) { + return nil + } + + creationDate, err := getCreationDate(ref) + if err != nil { + return fmt.Errorf("failed to get creation date for %s: %v", ref.Name(), err) + } + + mu.Lock() + defer mu.Unlock() + + if matchesFilters(tag, excludes, false) { + tagsKept = append(tagsKept, keptTag{ + refDate: refDate{ref, creationDate}, + keepReason: keepReasonExcluded, + }) + return nil + } + + tagsToConsiderRemoving = append(tagsToConsiderRemoving, refDate{ref, creationDate}) + return nil + }) + } + + if err := eg.Wait(); err != nil { + return repoSearchResult{}, err + } + + if len(tagsToConsiderRemoving) <= keepLatest { + keepLatest = len(tagsToConsiderRemoving) + } + + slices.SortFunc(tagsToConsiderRemoving, func(l, r refDate) int { + return l.date.Compare(r.date) * -1 + }) + + slices.SortFunc(tagsKept, func(l, r keptTag) int { + return l.refDate.date.Compare(r.refDate.date) * -1 + }) + + tagsToRemove := tagsToConsiderRemoving[keepLatest:] + for _, tag := range tagsToConsiderRemoving[:keepLatest] { + tagsKept = append(tagsKept, keptTag{refDate: tag, keepReason: keepReasonLatest}) + } + + return repoSearchResult{ + tagsToKeep: tagsKept, + tagsToRemove: tagsToRemove, + }, nil +} + +func matchesFilters(input string, filters []glob.Glob, matchesIfNoFilters bool) bool { + if len(filters) == 0 { + return matchesIfNoFilters + } + + for _, filter := range filters { + if filter.Match(input) { + return true + } + } + + return false +} + +func getCreationDate(ref name.Reference) (time.Time, error) { + img, err := getImage(ref) + if err != nil { + return time.Time{}, err + } + + configFile, err := img.ConfigFile() + if err != nil { + return time.Time{}, err + } + + return configFile.Created.Time, nil +} + +func getImage(ref name.Reference) (v1.Image, error) { + img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + if err != nil { + return nil, err + } + + return img, nil +} diff --git a/actions/clean-up-container-registry/tags.sh b/actions/clean-up-container-registry/tags.sh new file mode 100755 index 000000000..f37541ec3 --- /dev/null +++ b/actions/clean-up-container-registry/tags.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +readonly image_name=us.gcr.io/kubernetes-dev/infra-build +readonly exclude_tags="2024-06-17-*" +readonly tag_filter= +readonly keep_latest=5 +readonly dry_run="true" + +# List all image tags using crane +mapfile -t tags < <(crane ls "${image_name}") + +# Convert exclude_tags to an array +mapfile -t exclude_array <<< "${exclude_tags}" + +declare -A tags_to_consider_removing +declare -A tags_kept + +# Filter tags by the glob pattern, then sort them +for tag in "${tags[@]}"; do + # We want glob patching, that's the point! + # shellcheck disable=SC2053 + if [[ -z "${tag_filter}" ]] || [[ ${tag} == ${tag_filter} ]]; then + full_image_reference="${image_name}:${tag}" + + creation_date=$(crane config "${full_image_reference}" | jq -r '.created') + + for exclude in "${exclude_array[@]}"; do + if [[ $tag == ${exclude} ]]; then + tags_kept[${tag}]="${creation_date}" + continue 2 + fi + done + tags_to_consider_removing[${tag}]="${creation_date}" + fi +done + +# Sort tags by creation date +mapfile -t tags_to_consider_removing < <( + for tag in "${!tags_to_consider_removing[@]}"; do + echo "${tags_to_consider_removing["${tag}"]} ${tag}" + done | sort -k2 -r | awk '{print $1}' +) + +# Sort tags to keep by creation date +mapfile -t tags_kept < <( + for tag in "${!tags_kept[@]}"; do + echo "${tags_kept["${tag}"]} ${tag}" + done | sort -k2 -r | awk '{print $1}' +) + +# Determine tags to keep and tags to remove +tags_to_keep=("${tags_kept[@]: -${keep_latest}}") +tags_to_remove=("${tags_to_consider_removing[@]:${keep_latest}}") + +if [ "$dry_run" == "true" ]; then + echo "Dry-run mode: the following tags would be removed:" + for tag in "${tags_to_remove[@]}"; do + echo "- $tag" + done + + echo "...and the following tags would be kept:" + for tag in "${tags_to_keep[@]}"; do + echo "- $tag" + done +else + echo "Removing the following tags:" + printf "%s\n" "${tags_to_remove[@]}" + for tag in "${tags_to_remove[@]}"; do + crane delete "${image_name}:${tag}" + done +fi + +# Output removed tags +echo "::set-output name=removed_tags::$(IFS=,; echo "${tags_to_remove[*]}")" diff --git a/actions/push-to-gar-docker/action.yaml b/actions/push-to-gar-docker/action.yaml index 9fa523fa9..9ff9ace88 100644 --- a/actions/push-to-gar-docker/action.yaml +++ b/actions/push-to-gar-docker/action.yaml @@ -104,10 +104,23 @@ runs: ref: ${{ env.action_ref }} path: shared-workflows - - name: Get repository name - id: get-repository-name + - name: Set up variables + id: set-repo-variables shell: bash run: | + case "${{ inputs.environment }}" in + dev) + PROJECT="dev" + ;; + prod) + PROJECT="prod" + ;; + *) + echo "Invalid environment. Valid environments: dev, prod" + exit 1 + ;; + esac + REPO_NAME="${{ inputs.repository_name }}" if [ -z "$REPO_NAME" ]; then REPO_NAME="$(echo "${{ github.repository }}" | awk -F'/' '{print $2}')" @@ -117,19 +130,7 @@ runs: fi echo "repo_name=${REPO_NAME}" >> "${GITHUB_OUTPUT}" - - name: Resolve GCP project - id: resolve-project - shell: bash - run: | - if [[ "${{ inputs.environment }}" == "dev" ]]; then - PROJECT="grafanalabs-dev" - elif [[ "${{ inputs.environment }}" == "prod" ]]; then - PROJECT="grafanalabs-global" - else - echo "Invalid environment. Valid environment variable inputs: dev, prod" - exit 1 - fi - echo "project=${PROJECT}" | tee -a ${GITHUB_OUTPUT} + echo "image=${{ inputs.registry }}/${{ steps.resolve-project.outputs.project }}/docker-${REPO_NAME}-${PROJECT}/${{ inputs.image_name }}" >> "${GITHUB_OUTPUT}" - name: Login to GAR uses: ./shared-workflows/actions/login-to-gar @@ -140,7 +141,7 @@ runs: id: meta uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: - images: "${{ inputs.registry }}/${{ steps.resolve-project.outputs.project }}/docker-${{ steps.get-repository-name.outputs.repo_name }}-${{ inputs.environment }}/${{ inputs.image_name }}" + images: ${{ steps.set-repo-variables.outputs.image }} tags: ${{ inputs.tags }} - name: Set up Docker Buildx @@ -152,13 +153,22 @@ runs: uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 id: build with: - context: ${{ inputs.context }} + attests: type=provenance,mode=max build-args: ${{ inputs.build-args }} - push: ${{ inputs.push == 'true' }} - tags: ${{ steps.meta.outputs.tags }} + build-contexts: ${{ inputs.build-contexts }} cache-from: ${{ inputs.cache-from }} cache-to: ${{ inputs.cache-to }} + context: ${{ inputs.context }} file: ${{ inputs.file }} platforms: ${{ inputs.platforms }} + push: ${{ inputs.push == 'true' }} + sbom: true ssh: ${{ inputs.ssh }} - build-contexts: ${{ inputs.build-contexts }} + tags: ${{ steps.meta.outputs.tags }} + + - name: Attest build provenance + uses: actions/attest-build-provenance@5e9cb68e95676991667494a6a4e59b8a2f13e1d0 # v1.3.3 + with: + push-to-registry: ${{ inputs.push }} + subject-digest: ${{ steps.build-push.outputs.digest }} + subject-name: ${{ steps.set-repo-variables.outputs.image }} diff --git a/actions/syft-sbom-report/README.md b/actions/syft-sbom-report/README.md deleted file mode 100644 index 3d7a42cdd..000000000 --- a/actions/syft-sbom-report/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# syft-sbom-report - -Generate an SPDX SBOM Report and attached to Release Artifcats on Release Publish - -Example workflow: - -```yaml -name: syft-sbom-ci -on: - release: - types: [published] - workflow_dispatch: -jobs: - syft-sbom: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Anchore SBOM Action - uses: anchore/sbom-action@v0.15.10 - with: - artifact-name: ${{ github.event.repository.name }}-spdx.json -``` diff --git a/actions/syft-sbom-report/syft-sbom.yml b/actions/syft-sbom-report/syft-sbom.yml deleted file mode 100644 index 298fdb066..000000000 --- a/actions/syft-sbom-report/syft-sbom.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: syft-sbom-ci -on: - release: - types: [published] - workflow_dispatch: -jobs: - syft-sbom: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.0.0 - - name: Anchore SBOM Action - uses: anchore/sbom-action@ab5d7b5f48981941c4c5d6bf33aeb98fe3bae38c # 0.15.10 - with: - artifact-name: ${{ github.event.repository.name }}-spdx.json