Merge pull request #5186 from ncopa/kubelet-root-dir

Add support for setting kubelet root directory

cmd/controller/controller_test.go

@@ -74,6 +74,7 @@ Flags:
       --k0s-cloud-provider-update-frequency duration   the frequency of k0s-cloud-provider node updates (default 2m0s)
       --kube-controller-manager-extra-args string      extra args for kube-controller-manager
       --kubelet-extra-args string                      extra args for kubelet
+      --kubelet-root-dir string                        Kubelet root directory for k0s
       --labels strings                                 Node labels, list of key=value pairs
   -l, --logging stringToString                         Logging Levels for the different components (default [containerd=info,etcd=info,konnectivity-server=1,kube-apiserver=1,kube-controller-manager=1,kube-scheduler=1,kubelet=1])
       --no-taints                                      disable default taints for controller node

cmd/install/controller_test.go

@@ -72,6 +72,7 @@ Flags:
       --k0s-cloud-provider-update-frequency duration   the frequency of k0s-cloud-provider node updates (default 2m0s)
       --kube-controller-manager-extra-args string      extra args for kube-controller-manager
       --kubelet-extra-args string                      extra args for kubelet
+      --kubelet-root-dir string                        Kubelet root directory for k0s
       --labels strings                                 Node labels, list of key=value pairs
   -l, --logging stringToString                         Logging Levels for the different components (default [containerd=info,etcd=info,konnectivity-server=1,kube-apiserver=1,kube-controller-manager=1,kube-scheduler=1,kubelet=1])
       --no-taints                                      disable default taints for controller node

cmd/install/util.go

@@ -40,7 +40,7 @@ func cmdFlagsToArgs(cmd *cobra.Command) ([]string, error) {
 			switch f.Name {
 			case "env", "force":
 				return
-			case "data-dir", "token-file", "config":
+			case "data-dir", "kubelet-root-dir", "token-file", "config":
 				if absVal, err := filepath.Abs(val); err != nil {
 					err = fmt.Errorf("failed to convert --%s=%s to an absolute path: %w", f.Name, val, err)
 					errs = append(errs, err)

inttest/kubeletcertrotate/kubeletcertrotate_test.go

@@ -59,7 +59,7 @@ func (s *kubeletCertRotateSuite) SetupTest() {
 	s.Require().NoError(err)
 
 	// Start the workers using the join token
-	s.Require().NoError(s.RunWorkersWithToken(workerJoinToken))
+	s.Require().NoError(s.RunWorkersWithToken(workerJoinToken, "--kubelet-root-dir=/var/lib/kubelet"))
 
 	client, err := s.KubeClient(s.ControllerNode(0))
 	s.Require().NoError(err)
@@ -74,7 +74,7 @@ func (s *kubeletCertRotateSuite) SetupTest() {
 	workerSSH, err := s.SSH(s.Context(), s.WorkerNode(0))
 	s.Require().NoError(err)
 	s.T().Log("waiting to see kubelet rotating the client cert before triggering Plan creation")
-	_, err = workerSSH.ExecWithOutput(s.Context(), "inotifywait --no-dereference /var/lib/k0s/kubelet/pki/kubelet-client-current.pem")
+	_, err = workerSSH.ExecWithOutput(s.Context(), "inotifywait --no-dereference /var/lib/kubelet/pki/kubelet-client-current.pem")
 	s.Require().NoError(err)
 	output, err := workerSSH.ExecWithOutput(s.Context(), "k0s status -ojson")
 	s.Require().NoError(err)

inttest/reset/reset_test.go

@@ -42,7 +42,7 @@ func (s *suite) TestReset() {
 
 	if !s.Run("k0s gets up", func() {
 		s.Require().NoError(s.InitController(0, "--disable-components=konnectivity-server,metrics-server"))
-		s.Require().NoError(s.RunWorkers())
+		s.Require().NoError(s.RunWorkers("--kubelet-root-dir=/var/lib/kubelet"))
 
 		kc, err := s.KubeClient(s.ControllerNode(0))
 		s.Require().NoError(err)
@@ -59,6 +59,7 @@ func (s *suite) TestReset() {
 
 		s.NoError(ssh.Exec(ctx, "test -d /var/lib/k0s", common.SSHStreams{}), "/var/lib/k0s is not a directory")
 		s.NoError(ssh.Exec(ctx, "test -d /run/k0s", common.SSHStreams{}), "/run/k0s is not a directory")
+		s.NoError(ssh.Exec(ctx, "test -d /var/lib/kubelet", common.SSHStreams{}), "/var/lib/kubelet is not a directory")
 
 		s.NoError(ssh.Exec(ctx, "pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected some running containerd shims")
 	}) {
@@ -90,7 +91,7 @@ func (s *suite) TestReset() {
 		defer ssh.Disconnect()
 
 		streams, flushStreams := common.TestLogStreams(s.T(), "reset")
-		err = ssh.Exec(ctx, "k0s reset --debug", streams)
+		err = ssh.Exec(ctx, "k0s reset --debug --kubelet-root-dir=/var/lib/kubelet", streams)
 		flushStreams()
 		s.NoError(err, "k0s reset didn't exit cleanly")
 
@@ -104,6 +105,7 @@ func (s *suite) TestReset() {
 
 		// /var/lib/k0s is a mount point in the Docker container and can't be deleted, so it must be empty
 		s.NoError(ssh.Exec(ctx, `x="$(ls -A /var/lib/k0s)" && echo "$x" >&2 && [ -z "$x" ]`, common.SSHStreams{}), "/var/lib/k0s is not empty")
+		s.NoError(ssh.Exec(ctx, "! test -e /var/lib/kubelet", common.SSHStreams{}), "/var/lib/kubelet still exists")
 		s.NoError(ssh.Exec(ctx, "! test -e /run/k0s", common.SSHStreams{}), "/run/k0s still exists")
 		s.NoError(ssh.Exec(ctx, "! pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected no running containerd shims")
 	})

pkg/cleanup/cleanup.go

@@ -53,8 +53,9 @@ func NewConfig(debug bool, k0sVars *config.CfgVars, criSocketFlag string) (*Conf
 		},
 		&services{},
 		&directories{
-			dataDir: k0sVars.DataDir,
-			runDir:  k0sVars.RunDir,
+			dataDir:        k0sVars.DataDir,
+			kubeletRootDir: k0sVars.KubeletRootDir,
+			runDir:         k0sVars.RunDir,
 		},
 		&cni{},
 	}

pkg/cleanup/directories.go

@@ -28,8 +28,9 @@ import (
 )
 
 type directories struct {
-	dataDir string
-	runDir  string
+	dataDir        string
+	kubeletRootDir string
+	runDir         string
 }
 
 // Name returns the name of the step
@@ -70,7 +71,7 @@ func (d *directories) Run() error {
 			dataDirMounted = true
 			continue
 		}
-		if isUnderPath(v.Path, filepath.Join(d.dataDir, "kubelet")) || isUnderPath(v.Path, d.dataDir) {
+		if isUnderPath(v.Path, d.kubeletRootDir) || isUnderPath(v.Path, d.dataDir) {
 			logrus.Debugf("%v is mounted! attempting to unmount...", v.Path)
 			if err = mounter.Unmount(v.Path); err != nil {
 				// if we fail to unmount, try lazy unmount so
@@ -84,6 +85,11 @@ func (d *directories) Run() error {
 		}
 	}
 
+	logrus.Debugf("removing kubelet root dir (%s)", d.kubeletRootDir)
+	if err := os.RemoveAll(d.kubeletRootDir); err != nil {
+		return fmt.Errorf("failed to delete k0s kubelet root direcotory: %w", err)
+	}
+
 	if dataDirMounted {
 		logrus.Debugf("removing the contents of mounted data-dir (%s)", d.dataDir)
 	} else {

pkg/component/worker/kubelet.go

@@ -63,7 +63,6 @@ type Kubelet struct {
 	ExtraArgs           string
 	DualStackEnabled    bool
 
-	rootDir    string
 	configPath string
 	supervisor supervisor.Supervisor
 }
@@ -84,10 +83,9 @@ func (k *Kubelet) Init(_ context.Context) error {
 		}
 	}
 
-	k.rootDir = filepath.Join(k.K0sVars.DataDir, "kubelet")
-	err := dir.Init(k.rootDir, constant.DataDirMode)
+	err := dir.Init(k.K0sVars.KubeletRootDir, constant.DataDirMode)
 	if err != nil {
-		return fmt.Errorf("failed to create %s: %w", k.rootDir, err)
+		return fmt.Errorf("failed to create %s: %w", k.K0sVars.KubeletRootDir, err)
 	}
 
 	runDir := filepath.Join(k.K0sVars.RunDir, "kubelet")
@@ -133,12 +131,12 @@ func (k *Kubelet) Start(ctx context.Context) error {
 
 	logrus.Info("Starting kubelet")
 	args := stringmap.StringMap{
-		"--root-dir":        k.rootDir,
+		"--root-dir":        k.K0sVars.KubeletRootDir,
 		"--config":          k.configPath,
 		"--kubeconfig":      k.Kubeconfig,
 		"--v":               k.LogLevel,
 		"--runtime-cgroups": "/system.slice/containerd.service",
-		"--cert-dir":        filepath.Join(k.rootDir, "pki"),
+		"--cert-dir":        filepath.Join(k.K0sVars.KubeletRootDir, "pki"),
 	}
 
 	if len(k.Labels) > 0 {

pkg/component/worker/utils.go

@@ -144,7 +144,7 @@ func BootstrapKubeletKubeconfig(ctx context.Context, k0sVars *config.CfgVars, wo
 		return fmt.Errorf("wrong token type %s, expected type: kubelet-bootstrap", tokenType)
 	}
 
-	certDir := filepath.Join(k0sVars.DataDir, "kubelet", "pki")
+	certDir := filepath.Join(k0sVars.KubeletRootDir, "pki")
 	if err := dir.Init(certDir, constant.DataDirMode); err != nil {
 		return fmt.Errorf("failed to initialize kubelet certificate directory: %w", err)
 	}

pkg/config/cfgvars.go

@@ -47,6 +47,7 @@ type CfgVars struct {
 	BinDir                     string              // location for all pki related binaries
 	CertRootDir                string              // CertRootDir defines the root location for all pki related artifacts
 	DataDir                    string              // Data directory containing k0s state
+	KubeletRootDir             string              // Root directory for kubelet
 	EtcdCertDir                string              // EtcdCertDir contains etcd certificates
 	EtcdDataDir                string              // EtcdDataDir contains etcd state
 	KineSocketPath             string              // The unix socket path for kine
@@ -105,6 +106,12 @@ func WithCommand(cmd command) CfgVarOption {
 			c.DataDir = f
 		}
 
+		if f, err := flags.GetString("kubelet-root-dir"); err == nil && f != "" {
+			if f, err := filepath.Abs(f); err == nil {
+				c.KubeletRootDir = f
+			}
+		}
+
 		if f, err := flags.GetString("config"); err == nil && f != "" {
 			c.StartupConfigPath = f
 		}
@@ -137,6 +144,7 @@ func DefaultCfgVars() *CfgVars {
 // NewCfgVars returns a new CfgVars struct.
 func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) {
 	var dataDir string
+	var kubeletRootDir string
 
 	if len(dirs) > 0 {
 		dataDir = dirs[0]
@@ -146,6 +154,9 @@ func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) {
 		if val, err := cobraCmd.Flags().GetString("data-dir"); err == nil && val != "" {
 			dataDir = val
 		}
+		if val, err := cobraCmd.Flags().GetString("kubelet-root-dir"); err == nil && val != "" {
+			kubeletRootDir = val
+		}
 	}
 
 	if dataDir == "" {
@@ -158,6 +169,14 @@ func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) {
 		return nil, fmt.Errorf("invalid datadir: %w", err)
 	}
 
+	if kubeletRootDir == "" {
+		kubeletRootDir = filepath.Join(dataDir, "kubelet")
+	}
+	kubeletRootDir, err = filepath.Abs(kubeletRootDir)
+	if err != nil {
+		return nil, fmt.Errorf("invalid kubeletRootDir: %w", err)
+	}
+
 	var runDir string
 	if os.Geteuid() == 0 {
 		runDir = "/run/k0s"
@@ -180,6 +199,7 @@ func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) {
 		OCIBundleDir:               filepath.Join(dataDir, "images"),
 		CertRootDir:                certDir,
 		DataDir:                    dataDir,
+		KubeletRootDir:             kubeletRootDir,
 		EtcdCertDir:                filepath.Join(certDir, "etcd"),
 		EtcdDataDir:                filepath.Join(dataDir, "etcd"),
 		KineSocketPath:             filepath.Join(runDir, constant.KineSocket),

pkg/config/cfgvars_test.go

@@ -19,6 +19,7 @@ package config
 import (
 	"bytes"
 	"io"
+	"path/filepath"
 	"reflect"
 	"testing"
 
@@ -62,6 +63,7 @@ func TestWithCommand(t *testing.T) {
 	fakeFlags := &FakeFlagSet{
 		values: map[string]any{
 			"data-dir":              "/path/to/data",
+			"kubelet-root-dir":      "/path/to/kubelet",
 			"config":                "/path/to/config",
 			"status-socket":         "/path/to/socket",
 			"enable-dynamic-config": true,
@@ -79,8 +81,12 @@ func TestWithCommand(t *testing.T) {
 	c := &CfgVars{}
 	WithCommand(fakeCmd)(c)
 
+	dir, err := filepath.Abs("/path/to/kubelet")
+	assert.NoError(t, err)
+
 	assert.Same(t, in, c.stdin)
 	assert.Equal(t, "/path/to/data", c.DataDir)
+	assert.Equal(t, dir, c.KubeletRootDir)
 	assert.Equal(t, "/path/to/config", c.StartupConfigPath)
 	assert.Equal(t, "/path/to/socket", c.StatusSocketPath)
 	assert.True(t, c.EnableDynamicConfig)
@@ -149,6 +155,45 @@ func TestNewCfgVars_DataDir(t *testing.T) {
 	}
 }
 
+func TestNewCfgVars_KubeletRootDir(t *testing.T) {
+	tests := []struct {
+		name     string
+		fakeCmd  command
+		dirs     []string
+		expected *CfgVars
+	}{
+		{
+			name:     "default kubelet root dir",
+			fakeCmd:  &FakeCommand{flagSet: &FakeFlagSet{}},
+			expected: &CfgVars{KubeletRootDir: filepath.Join(constant.DataDirDefault, "kubelet")},
+		},
+		{
+			name: "default kubelet root dir when datadir set",
+			fakeCmd: &FakeCommand{
+				flagSet: &FakeFlagSet{values: map[string]any{"data-dir": "/path/to/data"}},
+			},
+			expected: &CfgVars{KubeletRootDir: "/path/to/data/kubelet"},
+		},
+		{
+			name: "custom kubelet root dir",
+			fakeCmd: &FakeCommand{
+				flagSet: &FakeFlagSet{values: map[string]any{"kubelet-root-dir": "/path/to/kubelet"}},
+			},
+			expected: &CfgVars{KubeletRootDir: "/path/to/kubelet"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c, err := NewCfgVars(tt.fakeCmd, tt.dirs...)
+			assert.NoError(t, err)
+			expected, err := filepath.Abs(tt.expected.KubeletRootDir)
+			assert.NoError(t, err)
+			assert.Equal(t, expected, c.KubeletRootDir)
+		})
+	}
+}
+
 func TestNodeConfig_Default(t *testing.T) {
 	oldDefaultPath := defaultConfigPath
 	defer func() { defaultConfigPath = oldDefaultPath }()

pkg/config/cli.go

@@ -200,6 +200,7 @@ func GetPersistentFlagSet() *pflag.FlagSet {
 	flagset.BoolVarP(&Debug, "debug", "d", false, "Debug logging (default: false)")
 	flagset.BoolVarP(&Verbose, "verbose", "v", false, "Verbose logging (default: false)")
 	flagset.String("data-dir", constant.DataDirDefault, "Data Directory for k0s. DO NOT CHANGE for an existing setup, things will break!")
+	flagset.String("kubelet-root-dir", "", "Kubelet root directory for k0s")
 	flagset.StringVar(&StatusSocket, "status-socket", "", "Full file path to the socket file. (default: <rundir>/status.sock)")
 	flagset.StringVar(&DebugListenOn, "debugListenOn", ":6060", "Http listenOn for Debug pprof handler")
 	return flagset