Skip to content

Commit

Permalink
Stage iptables binaries in a separate component
Browse files Browse the repository at this point in the history 
CPLB potentially will need to create some iptables rules for its
userspace proxy, hence this code should be in a separate component.

Signed-off-by: Juan-Luis de Sousa-Valadas Castaño <[email protected]>
  • Loading branch information
@juanluisvaladas
juanluisvaladas committed Nov 19, 2024
1 parent 06c59b3 commit ec6053a
Show file tree
Hide file tree
Showing 7 changed files with 132 additions and 85 deletions.
33 changes: 20 additions & 13 deletions cmd/worker/worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
k0slog "github.com/k0sproject/k0s/internal/pkg/log"
"github.com/k0sproject/k0s/internal/pkg/sysinfo"
"github.com/k0sproject/k0s/pkg/build"
"github.com/k0sproject/k0s/pkg/component/iptables"
"github.com/k0sproject/k0s/pkg/component/manager"
"github.com/k0sproject/k0s/pkg/component/prober"
"github.com/k0sproject/k0s/pkg/component/status"
Expand Down Expand Up @@ -147,21 +148,26 @@ func (c *Command) Start(ctx context.Context) error {
c.WorkerProfile = "default-windows"
}

componentManager.Add(ctx, &worker.Kubelet{
CRISocket: c.CriSocket,
EnableCloudProvider: c.CloudProvider,
K0sVars: c.K0sVars,
StaticPods: staticPods,
Kubeconfig: kubeletKubeconfigPath,
Configuration: *workerConfig.KubeletConfiguration.DeepCopy(),
LogLevel: c.LogLevels.Kubelet,
Labels: c.Labels,
Taints: c.Taints,
ExtraArgs: c.KubeletExtraArgs,
IPTablesMode: c.WorkerOptions.IPTablesMode,
DualStackEnabled: workerConfig.DualStackEnabled,
componentManager.Add(ctx, &iptables.Component{
IPTablesMode: c.WorkerOptions.IPTablesMode,
BinDir: c.K0sVars.BinDir,
})

componentManager.Add(ctx,
&worker.Kubelet{
CRISocket: c.CriSocket,
EnableCloudProvider: c.CloudProvider,
K0sVars: c.K0sVars,
StaticPods: staticPods,
Kubeconfig: kubeletKubeconfigPath,
Configuration: *workerConfig.KubeletConfiguration.DeepCopy(),
LogLevel: c.LogLevels.Kubelet,
Labels: c.Labels,
Taints: c.Taints,
ExtraArgs: c.KubeletExtraArgs,
DualStackEnabled: workerConfig.DualStackEnabled,
})

certManager := worker.NewCertificateManager(kubeletKubeconfigPath)

// if running inside a controller, status component is already running
Expand Down Expand Up @@ -196,6 +202,7 @@ func (c *Command) Start(ctx context.Context) error {
}

worker.KernelSetup()

err = componentManager.Start(ctx)
if err != nil {
return fmt.Errorf("failed to start worker components: %w", err)
Expand Down
85 changes: 83 additions & 2 deletions internal/pkg/iptablesutils/iptables.go → pkg/component/iptables/iptables.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
Copyright 2022 k0s authors
Copyright 2024 k0s authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
Expand All @@ -14,16 +14,21 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package iptablesutils
package iptables

import (
"bufio"
"context"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"

"github.com/k0sproject/k0s/pkg/assets"
"github.com/k0sproject/k0s/pkg/constant"
"github.com/sirupsen/logrus"
)

Expand All @@ -32,6 +37,82 @@ const (
ModeLegacy = "legacy"
)

type Component struct {
IPTablesMode string
BinDir string
}

func (c *Component) Init(_ context.Context) error {
log := logrus.WithField("component", constant.IptablesBinariesComponentName)
log.Info("Staging iptables binaries")
err, iptablesMode := extractIPTablesBinaries(c.BinDir, c.IPTablesMode)
if err != nil {
return err
}

c.IPTablesMode = iptablesMode
log.Infof("iptables mode: %s", c.IPTablesMode)
return nil
}

func (c *Component) Start(_ context.Context) error {
return nil
}

func (c *Component) Stop() error {
return nil
}

// extractIPTablesBinaries extracts the iptables binaries from the k0s binary and makes the symlinks
// to the backend detected by DetectHostIPTablesMode.
// extractIPTablesBinaries only works on linux, if called in another OS it will return an error.
func extractIPTablesBinaries(k0sBinDir string, iptablesMode string) (error, string) {
cmds := []string{"xtables-legacy-multi", "xtables-nft-multi"}
for _, cmd := range cmds {
err := assets.Stage(k0sBinDir, cmd, constant.BinDirMode)
if err != nil {
return err, ""
}
}
if iptablesMode == "" || iptablesMode == "auto" {
var err error
iptablesMode, err = DetectHostIPTablesMode(k0sBinDir)
if err != nil {
if kernelMajorVersion() < 5 {
iptablesMode = ModeLegacy
} else {
iptablesMode = ModeNFT
}
logrus.WithError(err).Infof("Failed to detect iptables mode, using iptables-%s by default", iptablesMode)
}
}
logrus.Infof("using iptables-%s", iptablesMode)
oldpath := fmt.Sprintf("xtables-%s-multi", iptablesMode)
for _, symlink := range []string{"iptables", "iptables-save", "iptables-restore", "ip6tables", "ip6tables-save", "ip6tables-restore"} {
symlinkPath := filepath.Join(k0sBinDir, symlink)

// remove if it exist and ignore error if it doesn't
_ = os.Remove(symlinkPath)

err := os.Symlink(oldpath, symlinkPath)
if err != nil {
return fmt.Errorf("failed to create symlink %s: %w", symlink, err), ""
}
}

return nil, iptablesMode
}
func kernelMajorVersion() byte {
if runtime.GOOS != "linux" {
return 0
}
data, err := os.ReadFile("/proc/sys/kernel/osrelease")
if err != nil {
return 0
}
return data[0] - '0'
}

// DetectHostIPTablesMode figure out whether iptables-legacy or iptables-nft is in use on the host.
// Follows the same logic as kube-proxy/kube-route.
// See: https://github.com/kubernetes-sigs/iptables-wrappers/blob/master/iptables-wrapper-installer.sh
Expand Down
46 changes: 23 additions & 23 deletions internal/pkg/iptablesutils/iptables_test.go → pkg/component/iptables/iptables_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
Copyright 2022 k0s authors
Copyright 2024 k0s authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
Expand All @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package iptablesutils_test
package iptables_test

import (
"fmt"
Expand All @@ -26,7 +26,7 @@ import (
"testing"

"github.com/k0sproject/k0s/internal/pkg/file"
"github.com/k0sproject/k0s/internal/pkg/iptablesutils"
"github.com/k0sproject/k0s/pkg/component/iptables"

"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -64,7 +64,7 @@ func TestDetectHostIPTablesMode(t *testing.T) {
t.Run("iptables_not_found", func(t *testing.T) {
binDir := t.TempDir()

_, err := iptablesutils.DetectHostIPTablesMode(binDir)
_, err := iptables.DetectHostIPTablesMode(binDir)

var execErr *exec.Error
require.ErrorAs(t, err, &execErr)
Expand All @@ -79,9 +79,9 @@ func TestDetectHostIPTablesMode(t *testing.T) {
strings.Repeat("echo KUBE-IPTABLES-HINT\n", 1),
)

mode, err := iptablesutils.DetectHostIPTablesMode(binDir)
mode, err := iptables.DetectHostIPTablesMode(binDir)
require.NoError(t, err)
assert.Equal(t, iptablesutils.ModeNFT, mode)
assert.Equal(t, iptables.ModeNFT, mode)
})

t.Run("xtables_legacy", func(t *testing.T) {
Expand All @@ -91,9 +91,9 @@ func TestDetectHostIPTablesMode(t *testing.T) {
strings.Repeat("echo KUBE-IPTABLES-HINT\n", 1),
)

mode, err := iptablesutils.DetectHostIPTablesMode(binDir)
mode, err := iptables.DetectHostIPTablesMode(binDir)
require.NoError(t, err)
assert.Equal(t, iptablesutils.ModeLegacy, mode)
assert.Equal(t, iptables.ModeLegacy, mode)
})

t.Run("xtables_nft_over_legacy", func(t *testing.T) {
Expand All @@ -108,9 +108,9 @@ func TestDetectHostIPTablesMode(t *testing.T) {
strings.Repeat("echo KUBE-IPTABLES-HINT\n", 3),
)

mode, err := iptablesutils.DetectHostIPTablesMode(binDir)
mode, err := iptables.DetectHostIPTablesMode(binDir)
require.NoError(t, err)
assert.Equal(t, iptablesutils.ModeNFT, mode)
assert.Equal(t, iptables.ModeNFT, mode)
})

t.Run("xtables_legacy_over_nft_more_entries", func(t *testing.T) {
Expand All @@ -124,9 +124,9 @@ func TestDetectHostIPTablesMode(t *testing.T) {
strings.Repeat("echo FOOBAR\n", 2),
)

mode, err := iptablesutils.DetectHostIPTablesMode(binDir)
mode, err := iptables.DetectHostIPTablesMode(binDir)
require.NoError(t, err)
assert.Equal(t, iptablesutils.ModeLegacy, mode)
assert.Equal(t, iptables.ModeLegacy, mode)
})

t.Run("fallback_to_iptables_if_xtables_nft_over_legacy_more_entries", func(t *testing.T) {
Expand All @@ -140,7 +140,7 @@ func TestDetectHostIPTablesMode(t *testing.T) {
strings.Repeat("echo FOOBAR\n", 1),
)

_, err := iptablesutils.DetectHostIPTablesMode(binDir)
_, err := iptables.DetectHostIPTablesMode(binDir)
var execErr *exec.Error
require.ErrorAs(t, err, &execErr)
assert.Equal(t, "iptables", execErr.Name)
Expand All @@ -152,27 +152,27 @@ func TestDetectHostIPTablesMode(t *testing.T) {
writeXtables(t, binDir, "nft", "exit 1", "exit 1")
writeXtables(t, binDir, "legacy", "exit 1", "echo KUBE-IPTABLES-HINT")

mode, err := iptablesutils.DetectHostIPTablesMode(binDir)
mode, err := iptables.DetectHostIPTablesMode(binDir)
require.NoError(t, err)
assert.Equal(t, iptablesutils.ModeLegacy, mode)
assert.Equal(t, iptables.ModeLegacy, mode)
})

t.Run("xtables_legacy_fails", func(t *testing.T) {
binDir := t.TempDir()
writeXtables(t, binDir, "nft", "exit 1", "echo KUBE-IPTABLES-HINT")
writeXtables(t, binDir, "legacy", "exit 1", "exit 1")

mode, err := iptablesutils.DetectHostIPTablesMode(binDir)
mode, err := iptables.DetectHostIPTablesMode(binDir)
require.NoError(t, err)
assert.Equal(t, iptablesutils.ModeNFT, mode)
assert.Equal(t, iptables.ModeNFT, mode)
})

t.Run("xtables_fails", func(t *testing.T) {
binDir := t.TempDir()
writeXtables(t, binDir, "nft", "exit 99", "exit 88")
writeXtables(t, binDir, "legacy", "exit 77", "exit 66")

_, err := iptablesutils.DetectHostIPTablesMode(binDir)
_, err := iptables.DetectHostIPTablesMode(binDir)
var composite interface{ Unwrap() []error }
require.ErrorAs(t, err, &composite, "No wrapped errors")
errs := composite.Unwrap()
Expand All @@ -190,23 +190,23 @@ func TestDetectHostIPTablesMode(t *testing.T) {
writeXtables(t, binDir, "legacy", "", "")

t.Run("iptables_legacy", func(t *testing.T) {
mode, err := iptablesutils.DetectHostIPTablesMode(binDir)
mode, err := iptables.DetectHostIPTablesMode(binDir)
require.NoError(t, err)
assert.Equal(t, iptablesutils.ModeLegacy, mode)
assert.Equal(t, iptables.ModeLegacy, mode)
})

writeScript(t, pathDir, "iptables", "echo foo-nf_tables-bar")

t.Run("iptables_nft", func(t *testing.T) {
mode, err := iptablesutils.DetectHostIPTablesMode(binDir)
mode, err := iptables.DetectHostIPTablesMode(binDir)
require.NoError(t, err)
assert.Equal(t, iptablesutils.ModeNFT, mode)
assert.Equal(t, iptables.ModeNFT, mode)
})

writeScript(t, pathDir, "iptables", "exit 1")

t.Run("iptables_broken", func(t *testing.T) {
_, err := iptablesutils.DetectHostIPTablesMode(binDir)
_, err := iptables.DetectHostIPTablesMode(binDir)
var exitErr *exec.ExitError
require.ErrorAs(t, err, &exitErr)
assert.Equal(t, 1, exitErr.ExitCode())
Expand Down
3 changes: 1 addition & 2 deletions pkg/component/worker/kernelsetup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,4 @@ limitations under the License.
package worker

// KernelSetup comment
func KernelSetup() {}
func KernelMajorVersion() byte { return 0 }
func KernelSetup() {}
9 changes: 0 additions & 9 deletions pkg/component/worker/kernelsetup_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,3 @@ func KernelSetup() {
enableSysCtl("net/bridge/bridge-nf-call-iptables")
enableSysCtl("net/bridge/bridge-nf-call-ip6tables")
}

// KernelMajorVersion returns the major version number of the running kernel
func KernelMajorVersion() byte {
data, err := os.ReadFile("/proc/sys/kernel/osrelease")
if err != nil {
return 0
}
return data[0] - '0'
}
Loading

0 comments on commit ec6053a

Please sign in to comment.