Add support for custom kubelet dir with --kubelet-root-dir

Storage drivers and others may hardcode /var/lib/kubelet which confilicts with the k0s default /var/lib/k0s/kubelet. Allow users to override the kubelet root directory with --kubelet-root-dir similar to the way they can override --data-dir. ref: https://cep.dev/posts/adventure-trying-change-kubelet-rootdir/ fixes #1842 Signed-off-by: Natanael Copa <[email protected]>
k0sproject · Nov 7, 2024 · 78c3f73 · 78c3f73
1 parent 56224d6
commit 78c3f73
Show file tree

Hide file tree

Showing 9 changed files with 68 additions and 9 deletions.
diff --git a/cmd/controller/controller_test.go b/cmd/controller/controller_test.go
@@ -74,6 +74,7 @@ Flags:
       --k0s-cloud-provider-update-frequency duration   the frequency of k0s-cloud-provider node updates (default 2m0s)
       --kube-controller-manager-extra-args string      extra args for kube-controller-manager
       --kubelet-extra-args string                      extra args for kubelet
+      --kubelet-root-dir string                        Kubelet root directory for k0s
       --labels strings                                 Node labels, list of key=value pairs
   -l, --logging stringToString                         Logging Levels for the different components (default [containerd=info,etcd=info,konnectivity-server=1,kube-apiserver=1,kube-controller-manager=1,kube-scheduler=1,kubelet=1])
       --no-taints                                      disable default taints for controller node

diff --git a/cmd/install/controller_test.go b/cmd/install/controller_test.go
@@ -72,6 +72,7 @@ Flags:
       --k0s-cloud-provider-update-frequency duration   the frequency of k0s-cloud-provider node updates (default 2m0s)
       --kube-controller-manager-extra-args string      extra args for kube-controller-manager
       --kubelet-extra-args string                      extra args for kubelet
+      --kubelet-root-dir string                        Kubelet root directory for k0s
       --labels strings                                 Node labels, list of key=value pairs
   -l, --logging stringToString                         Logging Levels for the different components (default [containerd=info,etcd=info,konnectivity-server=1,kube-apiserver=1,kube-controller-manager=1,kube-scheduler=1,kubelet=1])
       --no-taints                                      disable default taints for controller node

diff --git a/cmd/install/util.go b/cmd/install/util.go
@@ -40,7 +40,7 @@ func cmdFlagsToArgs(cmd *cobra.Command) ([]string, error) {
 			switch f.Name {
 			case "env", "force":
 				return
-			case "data-dir", "token-file", "config":
+			case "data-dir", "kubelet-root-dir", "token-file", "config":
 				if absVal, err := filepath.Abs(val); err != nil {
 					err = fmt.Errorf("failed to convert --%s=%s to an absolute path: %w", f.Name, val, err)
 					errs = append(errs, err)

diff --git a/pkg/cleanup/containers.go b/pkg/cleanup/containers.go
@@ -151,7 +151,7 @@ func (c *containers) stopAllContainers() error {
 		return fmt.Errorf("failed at listing pods %w", err)
 	}
 	if len(pods) > 0 {
-		if err := removeMount("kubelet/pods"); err != nil {
+		if err := removeMount(filepath.Join(c.Config.k0sVars.KubeletRootDir, "pods")); err != nil {
 			errs = append(errs, err)
 		}
 		if err := removeMount("run/netns"); err != nil {

diff --git a/pkg/component/worker/kubelet.go b/pkg/component/worker/kubelet.go
@@ -57,7 +57,6 @@ type Kubelet struct {
 	Configuration       kubeletv1beta1.KubeletConfiguration
 	StaticPods          StaticPods
 	LogLevel            string
-	dataDir             string
 	supervisor          supervisor.Supervisor
 	ClusterDNS          string
 	Labels              []string
@@ -123,10 +122,9 @@ func (k *Kubelet) Init(_ context.Context) error {
 		}
 	}
 
-	k.dataDir = filepath.Join(k.K0sVars.DataDir, "kubelet")
-	err := dir.Init(k.dataDir, constant.DataDirMode)
+	err := dir.Init(k.K0sVars.KubeletRootDir, constant.DataDirMode)
 	if err != nil {
-		return fmt.Errorf("failed to create %s: %w", k.dataDir, err)
+		return fmt.Errorf("failed to create %s: %w", k.K0sVars.KubeletRootDir, err)
 	}
 
 	return nil
@@ -179,12 +177,12 @@ func (k *Kubelet) Start(ctx context.Context) error {
 	kubeletConfigPath := filepath.Join(k.K0sVars.DataDir, "kubelet-config.yaml")
 
 	args := stringmap.StringMap{
-		"--root-dir":        k.dataDir,
+		"--root-dir":        k.K0sVars.KubeletRootDir,
 		"--config":          kubeletConfigPath,
 		"--kubeconfig":      k.Kubeconfig,
 		"--v":               k.LogLevel,
 		"--runtime-cgroups": "/system.slice/containerd.service",
-		"--cert-dir":        filepath.Join(k.dataDir, "pki"),
+		"--cert-dir":        filepath.Join(k.K0sVars.KubeletRootDir, "pki"),
 	}
 
 	if len(k.Labels) > 0 {

diff --git a/pkg/component/worker/utils.go b/pkg/component/worker/utils.go
@@ -144,7 +144,7 @@ func BootstrapKubeletKubeconfig(ctx context.Context, k0sVars *config.CfgVars, wo
 		return fmt.Errorf("wrong token type %s, expected type: kubelet-bootstrap", tokenType)
 	}
 
-	certDir := filepath.Join(k0sVars.DataDir, "kubelet", "pki")
+	certDir := filepath.Join(k0sVars.KubeletRootDir, "pki")
 	if err := dir.Init(certDir, constant.DataDirMode); err != nil {
 		return fmt.Errorf("failed to initialize kubelet certificate directory: %w", err)
 	}

diff --git a/pkg/config/cfgvars.go b/pkg/config/cfgvars.go
@@ -47,6 +47,7 @@ type CfgVars struct {
 	BinDir                     string              // location for all pki related binaries
 	CertRootDir                string              // CertRootDir defines the root location for all pki related artifacts
 	DataDir                    string              // Data directory containing k0s state
+	KubeletRootDir             string              // Root directory for kubelet
 	EtcdCertDir                string              // EtcdCertDir contains etcd certificates
 	EtcdDataDir                string              // EtcdDataDir contains etcd state
 	KineSocketPath             string              // The unix socket path for kine
@@ -105,6 +106,10 @@ func WithCommand(cmd command) CfgVarOption {
 			c.DataDir = f
 		}
 
+		if f, err := flags.GetString("kubelet-root-dir"); err == nil && f != "" {
+			c.KubeletRootDir = f
+		}
+
 		if f, err := flags.GetString("config"); err == nil && f != "" {
 			c.StartupConfigPath = f
 		}
@@ -137,6 +142,7 @@ func DefaultCfgVars() *CfgVars {
 // NewCfgVars returns a new CfgVars struct.
 func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) {
 	var dataDir string
+	var kubeletRootDir string
 
 	if len(dirs) > 0 {
 		dataDir = dirs[0]
@@ -146,6 +152,9 @@ func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) {
 		if val, err := cobraCmd.Flags().GetString("data-dir"); err == nil && val != "" {
 			dataDir = val
 		}
+		if val, err := cobraCmd.Flags().GetString("kubelet-root-dir"); err == nil && val != "" {
+			kubeletRootDir = val
+		}
 	}
 
 	if dataDir == "" {
@@ -158,6 +167,14 @@ func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) {
 		return nil, fmt.Errorf("invalid datadir: %w", err)
 	}
 
+	if kubeletRootDir == "" {
+		kubeletRootDir = filepath.Join(dataDir, "kubelet")
+	}
+	kubeletRootDir, err = filepath.Abs(kubeletRootDir)
+	if err != nil {
+		return nil, fmt.Errorf("invalid kubeletRootDir: %w", err)
+	}
+
 	var runDir string
 	if os.Geteuid() == 0 {
 		runDir = "/run/k0s"
@@ -180,6 +197,7 @@ func NewCfgVars(cobraCmd command, dirs ...string) (*CfgVars, error) {
 		OCIBundleDir:               filepath.Join(dataDir, "images"),
 		CertRootDir:                certDir,
 		DataDir:                    dataDir,
+		KubeletRootDir:             kubeletRootDir,
 		EtcdCertDir:                filepath.Join(certDir, "etcd"),
 		EtcdDataDir:                filepath.Join(dataDir, "etcd"),
 		KineSocketPath:             filepath.Join(runDir, constant.KineSocket),

diff --git a/pkg/config/cfgvars_test.go b/pkg/config/cfgvars_test.go
@@ -19,6 +19,7 @@ package config
 import (
 	"bytes"
 	"io"
+	"path/filepath"
 	"reflect"
 	"testing"
 
@@ -62,6 +63,7 @@ func TestWithCommand(t *testing.T) {
 	fakeFlags := &FakeFlagSet{
 		values: map[string]any{
 			"data-dir":              "/path/to/data",
+			"kubelet-root-dir":      "/path/to/kubelet",
 			"config":                "/path/to/config",
 			"status-socket":         "/path/to/socket",
 			"enable-dynamic-config": true,
@@ -81,6 +83,7 @@ func TestWithCommand(t *testing.T) {
 
 	assert.Same(t, in, c.stdin)
 	assert.Equal(t, "/path/to/data", c.DataDir)
+	assert.Equal(t, "/path/to/kubelet", c.KubeletRootDir)
 	assert.Equal(t, "/path/to/config", c.StartupConfigPath)
 	assert.Equal(t, "/path/to/socket", c.StatusSocketPath)
 	assert.True(t, c.EnableDynamicConfig)
@@ -149,6 +152,43 @@ func TestNewCfgVars_DataDir(t *testing.T) {
 	}
 }
 
+func TestNewCfgVars_KubeletRootDir(t *testing.T) {
+	tests := []struct {
+		name     string
+		fakeCmd  command
+		dirs     []string
+		expected *CfgVars
+	}{
+		{
+			name:     "default kubelet root dir",
+			fakeCmd:  &FakeCommand{flagSet: &FakeFlagSet{}},
+			expected: &CfgVars{KubeletRootDir: filepath.Join(constant.DataDirDefault, "kubelet")},
+		},
+		{
+			name: "default kubelet root dir when datadir set",
+			fakeCmd: &FakeCommand{
+				flagSet: &FakeFlagSet{values: map[string]any{"data-dir": "/path/to/data"}},
+			},
+			expected: &CfgVars{KubeletRootDir: "/path/to/data/kubelet"},
+		},
+		{
+			name: "custom kubelet root dir",
+			fakeCmd: &FakeCommand{
+				flagSet: &FakeFlagSet{values: map[string]any{"kubelet-root-dir": "/path/to/kubelet"}},
+			},
+			expected: &CfgVars{KubeletRootDir: "/path/to/kubelet"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c, err := NewCfgVars(tt.fakeCmd, tt.dirs...)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expected.KubeletRootDir, c.KubeletRootDir)
+		})
+	}
+}
+
 func TestNodeConfig_Default(t *testing.T) {
 	oldDefaultPath := defaultConfigPath
 	defer func() { defaultConfigPath = oldDefaultPath }()

diff --git a/pkg/config/cli.go b/pkg/config/cli.go
@@ -199,6 +199,7 @@ func GetPersistentFlagSet() *pflag.FlagSet {
 	flagset.BoolVarP(&Debug, "debug", "d", false, "Debug logging (default: false)")
 	flagset.BoolVarP(&Verbose, "verbose", "v", false, "Verbose logging (default: false)")
 	flagset.String("data-dir", constant.DataDirDefault, "Data Directory for k0s. DO NOT CHANGE for an existing setup, things will break!")
+	flagset.String("kubelet-root-dir", "", "Kubelet root directory for k0s")
 	flagset.StringVar(&StatusSocket, "status-socket", "", "Full file path to the socket file. (default: <rundir>/status.sock)")
 	flagset.StringVar(&DebugListenOn, "debugListenOn", ":6060", "Http listenOn for Debug pprof handler")
 	return flagset