Merge pull request #3891 from twz123/remove-kubeletconfig-component

Remove kubelet-config stack after deprecation in 1.26

cmd/controller/controller.go

@@ -469,7 +469,7 @@ func (c *command) start(ctx context.Context) error {
 			return err
 		}
 		clusterComponents.Add(ctx, reconciler)
-		clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars, adminClientFactory, nodeConfig))
+		clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars))
 	}
 
 	if !slices.Contains(c.DisableComponents, constant.SystemRbacComponentName) {

go.mod

@@ -29,7 +29,6 @@ require (
 	github.com/go-playground/validator/v10 v10.16.0
 	github.com/google/go-cmp v0.6.0
 	github.com/hashicorp/terraform-exec v0.20.0
-	github.com/imdario/mergo v0.3.16
 	github.com/k0sproject/bootloose v0.7.2
 	github.com/k0sproject/dig v0.2.0
 	github.com/k0sproject/version v0.4.2
@@ -176,6 +175,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/terraform-json v0.19.0 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/intel/goresctrl v0.3.0 // indirect
 	github.com/jmoiron/sqlx v1.3.5 // indirect

inttest/ap-ha3x3/ha3x3_test.go

@@ -15,24 +15,29 @@
 package ha3x3
 
 import (
+	"bytes"
+	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
-	apv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2"
 	apconst "github.com/k0sproject/k0s/pkg/autopilot/constant"
 	appc "github.com/k0sproject/k0s/pkg/autopilot/controller/plans/core"
 
 	"github.com/k0sproject/k0s/inttest/common"
 	aptest "github.com/k0sproject/k0s/inttest/common/autopilot"
 
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
+	"golang.org/x/exp/slices"
 )
 
 type ha3x3Suite struct {
 	common.BootlooseSuite
+	k0sUpdateVersion string
 }
 
 const haControllerConfig = `
@@ -93,9 +98,6 @@ func (s *ha3x3Suite) SetupTest() {
 // TestApply applies a well-formed `plan` yaml, and asserts that
 // all of the correct values across different objects + controllers are correct.
 func (s *ha3x3Suite) TestApply() {
-	k0sUpdateVersion := os.Getenv("K0S_UPDATE_TO_VERSION")
-	s.Require().NotEmpty(k0sUpdateVersion, "env var not set or empty: K0S_UPDATE_TO_VERSION")
-
 	planTemplate := `
 apiVersion: autopilot.k0sproject.io/v1beta2
 kind: Plan
@@ -106,7 +108,7 @@ spec:
   timestamp: now
   commands:
     - k0supdate:
-        version: ` + k0sUpdateVersion + `
+        version: ` + s.k0sUpdateVersion + `
         platforms:
           linux-amd64:
             url: http://localhost/dist/k0s-new
@@ -127,64 +129,158 @@ spec:
                   - worker2
 `
 
-	manifestFile := "/tmp/happy.yaml"
-	s.PutFileTemplate(s.ControllerNode(0), manifestFile, planTemplate, nil)
+	ctx := s.Context()
 
-	out, err := s.RunCommandController(0, fmt.Sprintf("/usr/local/bin/k0s kubectl apply -f %s", manifestFile))
-	s.T().Logf("kubectl apply output: '%s'", out)
+	sshController, err := s.SSH(ctx, s.ControllerNode(0))
 	s.Require().NoError(err)
+	defer sshController.Disconnect()
+
+	if version, err := s.GetK0sVersion(s.ControllerNode(0)); s.NoError(err, "Failed to get the base k0s version") {
+		hasOldStack := version != s.k0sUpdateVersion && (strings.HasPrefix(version, "v1.27.") || strings.HasPrefix(version, "v1.28."))
+		s.T().Logf("Base k0s version: %q, has old stack: %v", version, hasOldStack)
+		s.checkKubeletConfigStackResources(ctx, sshController, hasOldStack)
+	}
 
-	ssh, err := s.SSH(s.Context(), s.WorkerNode(0))
+	sshWorker, err := s.SSH(ctx, s.WorkerNode(0))
 	s.Require().NoError(err)
-	defer ssh.Disconnect()
-	out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V")
+	defer sshWorker.Disconnect()
+
+	iptablesModeBeforeUpdate, err := getIPTablesMode(ctx, sshWorker)
+	if !s.NoError(err) {
+		iptablesModeBeforeUpdate = ""
+	}
+
+	var createPlanOutput bytes.Buffer
+	err = sshController.Exec(ctx, "k0s kc create -f -", common.SSHStreams{
+		In:  strings.NewReader(planTemplate),
+		Out: &createPlanOutput,
+	})
 	s.Require().NoError(err)
-	iptablesVersionParts := strings.Split(out, " ")
-	iptablesModeBeforeUpdate := iptablesVersionParts[len(iptablesVersionParts)-1]
+	s.T().Log(strings.TrimSpace(createPlanOutput.String()))
 
 	client, err := s.AutopilotClient(s.ControllerNode(0))
 	s.Require().NoError(err)
-	s.NotEmpty(client)
 
 	// The plan has enough information to perform a successful update of k0s, so wait for it.
-	plan, err := aptest.WaitForPlanState(s.Context(), client, apconst.AutopilotName, appc.PlanCompleted)
+	s.T().Log("Waiting for autopilot plan to complete")
+	plan, err := aptest.WaitForPlanState(ctx, client, apconst.AutopilotName, appc.PlanCompleted)
 	s.Require().NoError(err)
+	s.T().Log("Autopilot plan completed")
 
 	// Ensure all state/status are completed
-	s.Equal(1, len(plan.Status.Commands))
-	cmd := plan.Status.Commands[0]
+	if s.Len(plan.Status.Commands, 1) {
+		cmd := plan.Status.Commands[0]
+		s.Equal(appc.PlanCompleted, cmd.State)
+		s.NotNil(cmd.K0sUpdate)
+		s.NotNil(cmd.K0sUpdate.Controllers)
+		s.NotNil(cmd.K0sUpdate.Workers)
+		s.Equal(appc.PlanCompleted, cmd.State)
+		s.NotNil(cmd.K0sUpdate)
+		s.NotNil(cmd.K0sUpdate.Controllers)
+		s.NotNil(cmd.K0sUpdate.Workers)
 
-	s.Equal(appc.PlanCompleted, cmd.State)
-	s.NotNil(cmd.K0sUpdate)
-	s.NotNil(cmd.K0sUpdate.Controllers)
-	s.NotNil(cmd.K0sUpdate.Workers)
+		if s.NotNil(cmd.K0sUpdate) {
+			s.Len(cmd.K0sUpdate.Controllers, s.ControllerCount)
+			for idx, controller := range cmd.K0sUpdate.Controllers {
+				s.Equal(appc.SignalCompleted, controller.State, "For controller %d", idx)
+			}
 
-	for _, group := range [][]apv1beta2.PlanCommandTargetStatus{cmd.K0sUpdate.Controllers, cmd.K0sUpdate.Workers} {
-		for _, node := range group {
-			s.Equal(appc.SignalCompleted, node.State)
+			s.Len(cmd.K0sUpdate.Workers, s.WorkerCount)
+			for idx, worker := range cmd.K0sUpdate.Workers {
+				s.Equal(appc.SignalCompleted, worker.State, "For worker %d", idx)
+			}
 		}
 	}
 
 	if version, err := s.GetK0sVersion(s.ControllerNode(0)); s.NoError(err) {
-		s.Equal(k0sUpdateVersion, version)
+		s.Equal(s.k0sUpdateVersion, version)
 	}
 
-	out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V")
-	s.Require().NoError(err)
-	iptablesVersionParts = strings.Split(out, " ")
-	iptablesModeAfterUpdate := iptablesVersionParts[len(iptablesVersionParts)-1]
-	s.Equal(iptablesModeBeforeUpdate, iptablesModeAfterUpdate)
+	if iptablesModeAfterUpdate, err := getIPTablesMode(ctx, sshWorker); s.NoError(err) {
+		s.Equal(iptablesModeBeforeUpdate, iptablesModeAfterUpdate)
+	}
+
+	for idx := 0; idx < s.ControllerCount; idx++ {
+		func() {
+			ssh, err := s.SSH(ctx, s.ControllerNode(idx))
+			s.Require().NoError(err)
+			defer ssh.Disconnect()
+			s.checkKubeletConfigComponentFolders(ctx, ssh)
+		}()
+	}
+
+	s.checkKubeletConfigStackResources(ctx, sshController, false)
+}
+
+func (s *ha3x3Suite) checkKubeletConfigComponentFolders(ctx context.Context, ssh *common.SSHConnection) {
+	var foundFiles bytes.Buffer
+	if !s.NoError(
+		ssh.Exec(ctx, "cd /var/lib/k0s/manifests/kubelet && find . -type f -print0", common.SSHStreams{Out: &foundFiles}),
+		"Failed to list kubelet manifest folder",
+	) {
+		return
+	}
+
+	files := strings.Split(strings.TrimSuffix(foundFiles.String(), "\x00"), "\x00")
+
+	// Check that removed.txt is present
+	if idx := slices.Index(files, "./removed.txt"); idx < 0 {
+		s.Failf("No removed.txt in kubelet manifests folder", "%v", files)
+	} else {
+		files = slices.Delete(files, idx, idx+1)
+	}
+
+	// Check that all other files are only disabled yaml files.
+	for _, file := range files {
+		match, err := filepath.Match("./kubelet-config.yaml.*.removed", file)
+		s.Require().NoError(err)
+		if !match {
+			s.Failf("Unknown file in kubelet manifest folder", "%s in %v", file, files)
+		}
+	}
+}
+
+func (s *ha3x3Suite) checkKubeletConfigStackResources(ctx context.Context, ssh *common.SSHConnection, exist bool) {
+	var out bytes.Buffer
+	err := ssh.Exec(ctx, "k0s kc get configmaps,roles,rolebindings -A -l 'k0s.k0sproject.io/stack=kubelet' -oname", common.SSHStreams{Out: &out})
+
+	if s.NoError(err) {
+		if exist {
+			s.NotEmpty(out.String())
+		} else {
+			s.Empty(out.String())
+		}
+	}
+}
+
+func getIPTablesMode(ctx context.Context, ssh *common.SSHConnection) (string, error) {
+	var out bytes.Buffer
+	err := ssh.Exec(ctx, "/var/lib/k0s/bin/iptables-save -V", common.SSHStreams{Out: &out})
+	if err != nil {
+		return "", err
+	}
+
+	version := out.String()
+	if parts := strings.Split(version, " "); len(parts) == 3 {
+		return parts[2], nil
+	}
+
+	return "", fmt.Errorf("expected something like %q, got %q", "iptables-save v1.8.9 (nf_tables)", version)
 }
 
 // TestHA3x3Suite sets up a suite using 3 controllers for quorum, and runs various
 // autopilot upgrade scenarios against them.
 func TestHA3x3Suite(t *testing.T) {
+	k0sUpdateVersion := os.Getenv("K0S_UPDATE_TO_VERSION")
+	require.NotEmpty(t, k0sUpdateVersion, "env var not set or empty: K0S_UPDATE_TO_VERSION")
+
 	suite.Run(t, &ha3x3Suite{
 		common.BootlooseSuite{
 			ControllerCount: 3,
 			WorkerCount:     3,
 			WithLB:          true,
 			LaunchMode:      common.LaunchModeOpenRC,
 		},
+		k0sUpdateVersion,
 	})
 }

pkg/applier/applier.go

@@ -19,7 +19,6 @@ package applier
 import (
 	"context"
 	"fmt"
-	"path"
 	"path/filepath"
 
 	"github.com/k0sproject/k0s/pkg/kubernetes"
@@ -35,6 +34,10 @@ import (
 // manifestFilePattern is the glob pattern that all applicable manifest files need to match.
 const manifestFilePattern = "*.yaml"
 
+func FindManifestFilesInDir(dir string) ([]string, error) {
+	return filepath.Glob(filepath.Join(dir, manifestFilePattern))
+}
+
 // Applier manages all the "static" manifests and applies them on the k8s API
 type Applier struct {
 	Name string
@@ -96,7 +99,7 @@ func (a *Applier) Apply(ctx context.Context) error {
 		return err
 	}
 
-	files, err := filepath.Glob(path.Join(a.Dir, manifestFilePattern))
+	files, err := FindManifestFilesInDir(a.Dir)
 	if err != nil {
 		return err
 	}