From 9852172ac011cea8c60a1244b08bad21004450cf Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Thu, 7 Nov 2024 13:02:20 +0100 Subject: [PATCH 1/3] Add support for custom kubelet dir with --kubelet-root-dir Storage drivers and others may hardcode /var/lib/kubelet which confilicts with the k0s default /var/lib/k0s/kubelet. Allow users to override the kubelet root directory with --kubelet-root-dir similar to the way they can override --data-dir. ref: https://cep.dev/posts/adventure-trying-change-kubelet-rootdir/ fixes: https://github.com/k0sproject/k0s/issues/1842 Signed-off-by: Natanael Copa --- cmd/controller/controller_test.go | 1 + cmd/install/controller_test.go | 1 + cmd/install/util.go | 2 +- pkg/cleanup/cleanup.go | 5 ++-- pkg/cleanup/directories.go | 12 ++++++--- pkg/component/worker/kubelet.go | 10 +++----- pkg/component/worker/utils.go | 2 +- pkg/config/cfgvars.go | 18 +++++++++++++ pkg/config/cfgvars_test.go | 42 +++++++++++++++++++++++++++++++ pkg/config/cli.go | 1 + 10 files changed, 81 insertions(+), 13 deletions(-) diff --git a/cmd/controller/controller_test.go b/cmd/controller/controller_test.go index 31b761479088..2f3f270775e8 100644 --- a/cmd/controller/controller_test.go +++ b/cmd/controller/controller_test.go @@ -74,6 +74,7 @@ Flags: --k0s-cloud-provider-update-frequency duration the frequency of k0s-cloud-provider node updates (default 2m0s) --kube-controller-manager-extra-args string extra args for kube-controller-manager --kubelet-extra-args string extra args for kubelet + --kubelet-root-dir string Kubelet root directory for k0s --labels strings Node labels, list of key=value pairs -l, --logging stringToString Logging Levels for the different components (default [containerd=info,etcd=info,konnectivity-server=1,kube-apiserver=1,kube-controller-manager=1,kube-scheduler=1,kubelet=1]) --no-taints disable default taints for controller node diff --git a/cmd/install/controller_test.go b/cmd/install/controller_test.go index 0571ff991fb7..bfab32f960c3 100644 --- a/cmd/install/controller_test.go +++ b/cmd/install/controller_test.go @@ -72,6 +72,7 @@ Flags: --k0s-cloud-provider-update-frequency duration the frequency of k0s-cloud-provider node updates (default 2m0s) --kube-controller-manager-extra-args string extra args for kube-controller-manager --kubelet-extra-args string extra args for kubelet + --kubelet-root-dir string Kubelet root directory for k0s --labels strings Node labels, list of key=value pairs -l, --logging stringToString Logging Levels for the different components (default [containerd=info,etcd=info,konnectivity-server=1,kube-apiserver=1,kube-controller-manager=1,kube-scheduler=1,kubelet=1]) --no-taints disable default taints for controller node diff --git a/cmd/install/util.go b/cmd/install/util.go index 6321c2a3c0e8..6ba8e661c0bd 100644 --- a/cmd/install/util.go +++ b/cmd/install/util.go @@ -40,7 +40,7 @@ func cmdFlagsToArgs(cmd *cobra.Command) ([]string, error) { switch f.Name { case "env", "force": return - case "data-dir", "token-file", "config": + case "data-dir", "kubelet-root-dir", "token-file", "config": if absVal, err := filepath.Abs(val); err != nil { err = fmt.Errorf("failed to convert --%s=%s to an absolute path: %w", f.Name, val, err) errs = append(errs, err) diff --git a/pkg/cleanup/cleanup.go b/pkg/cleanup/cleanup.go index c07cbba24321..a2d97571ac97 100644 --- a/pkg/cleanup/cleanup.go +++ b/pkg/cleanup/cleanup.go @@ -53,8 +53,9 @@ func NewConfig(debug bool, k0sVars *config.CfgVars, criSocketFlag string) (*Conf }, &services{}, &directories{ - dataDir: k0sVars.DataDir, - runDir: k0sVars.RunDir, + dataDir: k0sVars.DataDir, + kubeletRootDir: k0sVars.KubeletRootDir, + runDir: k0sVars.RunDir, }, &cni{}, } diff --git a/pkg/cleanup/directories.go b/pkg/cleanup/directories.go index b1c11f849405..096c8a53ebc5 100644 --- a/pkg/cleanup/directories.go +++ b/pkg/cleanup/directories.go @@ -28,8 +28,9 @@ import ( ) type directories struct { - dataDir string - runDir string + dataDir string + kubeletRootDir string + runDir string } // Name returns the name of the step @@ -70,7 +71,7 @@ func (d *directories) Run() error { dataDirMounted = true continue } - if isUnderPath(v.Path, filepath.Join(d.dataDir, "kubelet")) || isUnderPath(v.Path, d.dataDir) { + if isUnderPath(v.Path, d.kubeletRootDir) || isUnderPath(v.Path, d.dataDir) { logrus.Debugf("%v is mounted! attempting to unmount...", v.Path) if err = mounter.Unmount(v.Path); err != nil { // if we fail to unmount, try lazy unmount so @@ -84,6 +85,11 @@ func (d *directories) Run() error { } } + logrus.Debugf("removing kubelet root dir (%s)", d.kubeletRootDir) + if err := os.RemoveAll(d.kubeletRootDir); err != nil { + return fmt.Errorf("failed to delete k0s kubelet root direcotory: %w", err) + } + if dataDirMounted { logrus.Debugf("removing the contents of mounted data-dir (%s)", d.dataDir) } else { diff --git a/pkg/component/worker/kubelet.go b/pkg/component/worker/kubelet.go index 8a0465808674..332edd1fa3ca 100644 --- a/pkg/component/worker/kubelet.go +++ b/pkg/component/worker/kubelet.go @@ -63,7 +63,6 @@ type Kubelet struct { ExtraArgs string DualStackEnabled bool - rootDir string configPath string supervisor supervisor.Supervisor } @@ -84,10 +83,9 @@ func (k *Kubelet) Init(_ context.Context) error { } } - k.rootDir = filepath.Join(k.K0sVars.DataDir, "kubelet") - err := dir.Init(k.rootDir, constant.DataDirMode) + err := dir.Init(k.K0sVars.KubeletRootDir, constant.DataDirMode) if err != nil { - return fmt.Errorf("failed to create %s: %w", k.rootDir, err) + return fmt.Errorf("failed to create %s: %w", k.K0sVars.KubeletRootDir, err) } runDir := filepath.Join(k.K0sVars.RunDir, "kubelet") @@ -133,12 +131,12 @@ func (k *Kubelet) Start(ctx context.Context) error { logrus.Info("Starting kubelet") args := stringmap.StringMap{ - "--root-dir": k.rootDir, + "--root-dir": k.K0sVars.KubeletRootDir, "--config": k.configPath, "--kubeconfig": k.Kubeconfig, "--v": k.LogLevel, "--runtime-cgroups": "/system.slice/containerd.service", - "--cert-dir": filepath.Join(k.rootDir, "pki"), + "--cert-dir": filepath.Join(k.K0sVars.KubeletRootDir, "pki"), } if len(k.Labels) > 0 { diff --git a/pkg/component/worker/utils.go b/pkg/component/worker/utils.go index 655105552995..369681cc9cd5 100644 --- a/pkg/component/worker/utils.go +++ b/pkg/component/worker/utils.go @@ -144,7 +144,7 @@ func BootstrapKubeletKubeconfig(ctx context.Context, k0sVars *config.CfgVars, wo return fmt.Errorf("wrong token type %s, expected type: kubelet-bootstrap", tokenType) } - certDir := filepath.Join(k0sVars.DataDir, "kubelet", "pki") + certDir := filepath.Join(k0sVars.KubeletRootDir, "pki") if err := dir.Init(certDir, constant.DataDirMode); err != nil { return fmt.Errorf("failed to initialize kubelet certificate directory: %w", err) } diff --git a/pkg/config/cfgvars.go b/pkg/config/cfgvars.go index 947252c069cb..f8e720209473 100644 --- a/pkg/config/cfgvars.go +++ b/pkg/config/cfgvars.go @@ -47,6 +47,7 @@ type CfgVars struct { BinDir string // location for all pki related binaries CertRootDir string // CertRootDir defines the root location for all pki related artifacts DataDir string // Data directory containing k0s state + KubeletRootDir string // Root directory for kubelet EtcdCertDir string // EtcdCertDir contains etcd certificates EtcdDataDir string // EtcdDataDir contains etcd state KineSocketPath string // The unix socket path for kine @@ -105,6 +106,10 @@ func WithCommand(cmd command) CfgVarOption { c.DataDir = f } + if f, err := flags.GetString("kubelet-root-dir"); err == nil && f != "" { + c.KubeletRootDir = f + } + if f, err := flags.GetString("config"); err == nil && f != "" { c.StartupConfigPath = f } @@ -137,6 +142,7 @@ func DefaultCfgVars() *CfgVars { // NewCfgVars returns a new CfgVars struct. func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) { var dataDir string + var kubeletRootDir string if len(dirs) > 0 { dataDir = dirs[0] @@ -146,6 +152,9 @@ func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) { if val, err := cobraCmd.Flags().GetString("data-dir"); err == nil && val != "" { dataDir = val } + if val, err := cobraCmd.Flags().GetString("kubelet-root-dir"); err == nil && val != "" { + kubeletRootDir = val + } } if dataDir == "" { @@ -158,6 +167,14 @@ func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) { return nil, fmt.Errorf("invalid datadir: %w", err) } + if kubeletRootDir == "" { + kubeletRootDir = filepath.Join(dataDir, "kubelet") + } + kubeletRootDir, err = filepath.Abs(kubeletRootDir) + if err != nil { + return nil, fmt.Errorf("invalid kubeletRootDir: %w", err) + } + var runDir string if os.Geteuid() == 0 { runDir = "/run/k0s" @@ -180,6 +197,7 @@ func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) { OCIBundleDir: filepath.Join(dataDir, "images"), CertRootDir: certDir, DataDir: dataDir, + KubeletRootDir: kubeletRootDir, EtcdCertDir: filepath.Join(certDir, "etcd"), EtcdDataDir: filepath.Join(dataDir, "etcd"), KineSocketPath: filepath.Join(runDir, constant.KineSocket), diff --git a/pkg/config/cfgvars_test.go b/pkg/config/cfgvars_test.go index 747f1c291f1b..f66f63cdaecf 100644 --- a/pkg/config/cfgvars_test.go +++ b/pkg/config/cfgvars_test.go @@ -19,6 +19,7 @@ package config import ( "bytes" "io" + "path/filepath" "reflect" "testing" @@ -62,6 +63,7 @@ func TestWithCommand(t *testing.T) { fakeFlags := &FakeFlagSet{ values: map[string]any{ "data-dir": "/path/to/data", + "kubelet-root-dir": "/path/to/kubelet", "config": "/path/to/config", "status-socket": "/path/to/socket", "enable-dynamic-config": true, @@ -81,6 +83,7 @@ func TestWithCommand(t *testing.T) { assert.Same(t, in, c.stdin) assert.Equal(t, "/path/to/data", c.DataDir) + assert.Equal(t, "/path/to/kubelet", c.KubeletRootDir) assert.Equal(t, "/path/to/config", c.StartupConfigPath) assert.Equal(t, "/path/to/socket", c.StatusSocketPath) assert.True(t, c.EnableDynamicConfig) @@ -149,6 +152,45 @@ func TestNewCfgVars_DataDir(t *testing.T) { } } +func TestNewCfgVars_KubeletRootDir(t *testing.T) { + tests := []struct { + name string + fakeCmd command + dirs []string + expected *CfgVars + }{ + { + name: "default kubelet root dir", + fakeCmd: &FakeCommand{flagSet: &FakeFlagSet{}}, + expected: &CfgVars{KubeletRootDir: filepath.Join(constant.DataDirDefault, "kubelet")}, + }, + { + name: "default kubelet root dir when datadir set", + fakeCmd: &FakeCommand{ + flagSet: &FakeFlagSet{values: map[string]any{"data-dir": "/path/to/data"}}, + }, + expected: &CfgVars{KubeletRootDir: "/path/to/data/kubelet"}, + }, + { + name: "custom kubelet root dir", + fakeCmd: &FakeCommand{ + flagSet: &FakeFlagSet{values: map[string]any{"kubelet-root-dir": "/path/to/kubelet"}}, + }, + expected: &CfgVars{KubeletRootDir: "/path/to/kubelet"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, err := NewCfgVars(tt.fakeCmd, tt.dirs...) + assert.NoError(t, err) + expected, err := filepath.Abs(tt.expected.KubeletRootDir) + assert.NoError(t, err) + assert.Equal(t, expected, c.KubeletRootDir) + }) + } +} + func TestNodeConfig_Default(t *testing.T) { oldDefaultPath := defaultConfigPath defer func() { defaultConfigPath = oldDefaultPath }() diff --git a/pkg/config/cli.go b/pkg/config/cli.go index 2cca62e56b25..91cf8368ebbd 100644 --- a/pkg/config/cli.go +++ b/pkg/config/cli.go @@ -199,6 +199,7 @@ func GetPersistentFlagSet() *pflag.FlagSet { flagset.BoolVarP(&Debug, "debug", "d", false, "Debug logging (default: false)") flagset.BoolVarP(&Verbose, "verbose", "v", false, "Verbose logging (default: false)") flagset.String("data-dir", constant.DataDirDefault, "Data Directory for k0s. DO NOT CHANGE for an existing setup, things will break!") + flagset.String("kubelet-root-dir", "", "Kubelet root directory for k0s") flagset.StringVar(&StatusSocket, "status-socket", "", "Full file path to the socket file. (default: /status.sock)") flagset.StringVar(&DebugListenOn, "debugListenOn", ":6060", "Http listenOn for Debug pprof handler") return flagset From e3ab3c1968d442228b05f4e4f6e99aae0b251862 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Thu, 7 Nov 2024 21:13:08 +0100 Subject: [PATCH 2/3] Use --kubelet-root-dir in a couple of inttests Use --kubelet-root-dir for the inttests: - kubeletcertrotate - reset Signed-off-by: Natanael Copa --- inttest/kubeletcertrotate/kubeletcertrotate_test.go | 4 ++-- inttest/reset/reset_test.go | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/inttest/kubeletcertrotate/kubeletcertrotate_test.go b/inttest/kubeletcertrotate/kubeletcertrotate_test.go index cc6772c48628..39cb94a8a474 100644 --- a/inttest/kubeletcertrotate/kubeletcertrotate_test.go +++ b/inttest/kubeletcertrotate/kubeletcertrotate_test.go @@ -59,7 +59,7 @@ func (s *kubeletCertRotateSuite) SetupTest() { s.Require().NoError(err) // Start the workers using the join token - s.Require().NoError(s.RunWorkersWithToken(workerJoinToken)) + s.Require().NoError(s.RunWorkersWithToken(workerJoinToken, "--kubelet-root-dir=/var/lib/kubelet")) client, err := s.KubeClient(s.ControllerNode(0)) s.Require().NoError(err) @@ -74,7 +74,7 @@ func (s *kubeletCertRotateSuite) SetupTest() { workerSSH, err := s.SSH(s.Context(), s.WorkerNode(0)) s.Require().NoError(err) s.T().Log("waiting to see kubelet rotating the client cert before triggering Plan creation") - _, err = workerSSH.ExecWithOutput(s.Context(), "inotifywait --no-dereference /var/lib/k0s/kubelet/pki/kubelet-client-current.pem") + _, err = workerSSH.ExecWithOutput(s.Context(), "inotifywait --no-dereference /var/lib/kubelet/pki/kubelet-client-current.pem") s.Require().NoError(err) output, err := workerSSH.ExecWithOutput(s.Context(), "k0s status -ojson") s.Require().NoError(err) diff --git a/inttest/reset/reset_test.go b/inttest/reset/reset_test.go index d6e5d2a6712d..054ab6a060ff 100644 --- a/inttest/reset/reset_test.go +++ b/inttest/reset/reset_test.go @@ -42,7 +42,7 @@ func (s *suite) TestReset() { if !s.Run("k0s gets up", func() { s.Require().NoError(s.InitController(0, "--disable-components=konnectivity-server,metrics-server")) - s.Require().NoError(s.RunWorkers()) + s.Require().NoError(s.RunWorkers("--kubelet-root-dir=/var/lib/kubelet")) kc, err := s.KubeClient(s.ControllerNode(0)) s.Require().NoError(err) @@ -59,6 +59,7 @@ func (s *suite) TestReset() { s.NoError(ssh.Exec(ctx, "test -d /var/lib/k0s", common.SSHStreams{}), "/var/lib/k0s is not a directory") s.NoError(ssh.Exec(ctx, "test -d /run/k0s", common.SSHStreams{}), "/run/k0s is not a directory") + s.NoError(ssh.Exec(ctx, "test -d /var/lib/kubelet", common.SSHStreams{}), "/var/lib/kubelet is not a directory") s.NoError(ssh.Exec(ctx, "pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected some running containerd shims") }) { @@ -90,7 +91,7 @@ func (s *suite) TestReset() { defer ssh.Disconnect() streams, flushStreams := common.TestLogStreams(s.T(), "reset") - err = ssh.Exec(ctx, "k0s reset --debug", streams) + err = ssh.Exec(ctx, "k0s reset --debug --kubelet-root-dir=/var/lib/kubelet", streams) flushStreams() s.NoError(err, "k0s reset didn't exit cleanly") @@ -104,6 +105,7 @@ func (s *suite) TestReset() { // /var/lib/k0s is a mount point in the Docker container and can't be deleted, so it must be empty s.NoError(ssh.Exec(ctx, `x="$(ls -A /var/lib/k0s)" && echo "$x" >&2 && [ -z "$x" ]`, common.SSHStreams{}), "/var/lib/k0s is not empty") + s.NoError(ssh.Exec(ctx, "! test -e /var/lib/kubelet", common.SSHStreams{}), "/var/lib/kubelet still exists") s.NoError(ssh.Exec(ctx, "! test -e /run/k0s", common.SSHStreams{}), "/run/k0s still exists") s.NoError(ssh.Exec(ctx, "! pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected no running containerd shims") }) From 8c3073726caf93bfed217d4382450337867ce177 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Thu, 19 Dec 2024 15:09:17 +0100 Subject: [PATCH 3/3] Make cfgvars unit test pass on windows Signed-off-by: Natanael Copa --- pkg/config/cfgvars.go | 4 +++- pkg/config/cfgvars_test.go | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/config/cfgvars.go b/pkg/config/cfgvars.go index f8e720209473..3acfc10734f6 100644 --- a/pkg/config/cfgvars.go +++ b/pkg/config/cfgvars.go @@ -107,7 +107,9 @@ func WithCommand(cmd command) CfgVarOption { } if f, err := flags.GetString("kubelet-root-dir"); err == nil && f != "" { - c.KubeletRootDir = f + if f, err := filepath.Abs(f); err == nil { + c.KubeletRootDir = f + } } if f, err := flags.GetString("config"); err == nil && f != "" { diff --git a/pkg/config/cfgvars_test.go b/pkg/config/cfgvars_test.go index f66f63cdaecf..f1e7e5a50e6f 100644 --- a/pkg/config/cfgvars_test.go +++ b/pkg/config/cfgvars_test.go @@ -81,9 +81,12 @@ func TestWithCommand(t *testing.T) { c := &CfgVars{} WithCommand(fakeCmd)(c) + dir, err := filepath.Abs("/path/to/kubelet") + assert.NoError(t, err) + assert.Same(t, in, c.stdin) assert.Equal(t, "/path/to/data", c.DataDir) - assert.Equal(t, "/path/to/kubelet", c.KubeletRootDir) + assert.Equal(t, dir, c.KubeletRootDir) assert.Equal(t, "/path/to/config", c.StartupConfigPath) assert.Equal(t, "/path/to/socket", c.StatusSocketPath) assert.True(t, c.EnableDynamicConfig)