From fa6ead508d9109b90dc924d7b37895251caeb2bb Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Mon, 23 Sep 2024 21:49:51 +0300 Subject: [PATCH 1/6] Add DenyPSALabel admission plugin Signed-off-by: galal-hussein --- pkg/admission/denypsalabel/admission.go | 93 +++++++++++++++++++++++++ pkg/cli/cmds/server.go | 7 ++ pkg/cli/server/server.go | 1 + pkg/daemons/config/types.go | 1 + pkg/daemons/control/server.go | 10 +++ 5 files changed, 112 insertions(+) create mode 100644 pkg/admission/denypsalabel/admission.go diff --git a/pkg/admission/denypsalabel/admission.go b/pkg/admission/denypsalabel/admission.go new file mode 100644 index 000000000000..132b98556915 --- /dev/null +++ b/pkg/admission/denypsalabel/admission.go @@ -0,0 +1,93 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package denypsalabel + +import ( + "context" + "fmt" + "io" + "strings" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apiserver/pkg/admission" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/apis/core" +) + +const ( + // PluginName is the name of this admission controller plugin + PluginName = "DenyPSALabel" + PSALabelPrefix = "pod-security.kubernetes.io" +) + +// Register registers a plugin +func Register(plugins *admission.Plugins) { + plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { + plugin := newPlugin() + return plugin, nil + }) +} + +// externalIPsDenierPlugin holds state for and implements the admission plugin. +type psaLabelDenialPlugin struct { + *admission.Handler +} + +var _ admission.Interface = &psaLabelDenialPlugin{} +var _ admission.ValidationInterface = &psaLabelDenialPlugin{} + +// newPlugin creates a new admission plugin. +func newPlugin() *psaLabelDenialPlugin { + return &psaLabelDenialPlugin{ + Handler: admission.NewHandler(admission.Create, admission.Update), + } +} + +// Admit ensures that modifications of the Service.Spec.ExternalIPs field are +// denied +func (plug *psaLabelDenialPlugin) Validate(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces) error { + if attr.GetResource().GroupResource() != core.Resource("namespaces") { + return nil + } + + if len(attr.GetSubresource()) != 0 { + return nil + } + + // if we can't convert then we don't handle this object so just return + newNS, ok := attr.GetObject().(*core.Namespace) + if !ok { + klog.V(3).Infof("Expected Namespace resource, got: %v", attr.GetKind()) + return errors.NewInternalError(fmt.Errorf("Expected Namespace resource, got: %v", attr.GetKind())) + } + + if !isPSALabel(newNS) { + return nil + } + + klog.V(4).Infof("Denying use of PSA label on namespace %s", newNS.Name) + return admission.NewForbidden(attr, fmt.Errorf("Denying use of PSA label on Namespace")) +} + +func isPSALabel(newNS *core.Namespace) bool { + for labelName := range newNS.Labels { + if strings.HasPrefix(labelName, PSALabelPrefix) { + return true + } + } + return false +} diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index cfc684f169f3..404ee130d6fe 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -109,6 +109,7 @@ type Server struct { EtcdS3Timeout time.Duration EtcdS3Insecure bool ServiceLBNamespace string + DenyPSALabel bool } var ( @@ -586,6 +587,12 @@ var ServerFlags = []cli.Flag{ Usage: "(flags) Customized flag for kube-cloud-controller-manager process", Value: &ServerConfig.ExtraCloudControllerArgs, }, + &cli.BoolFlag{ + Name: "deny-psa-label", + Usage: "(experimental) Deny modifying namespaces with PSA security label", + Hidden: true, + Destination: &ServerConfig.DenyPSALabel, + }, } func NewServerCommand(action func(*cli.Context) error) cli.Command { diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 698d04ec49e3..fd73c1a9bd25 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -180,6 +180,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.SupervisorMetrics = cfg.SupervisorMetrics serverConfig.ControlConfig.VLevel = cmds.LogConfig.VLevel serverConfig.ControlConfig.VModule = cmds.LogConfig.VModule + serverConfig.ControlConfig.DenyPSALabel = cfg.DenyPSALabel if !cfg.EtcdDisableSnapshots || cfg.ClusterReset { serverConfig.ControlConfig.EtcdSnapshotCompress = cfg.EtcdSnapshotCompress diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 93e354e1962c..57bc84ea17a9 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -245,6 +245,7 @@ type Control struct { ServerNodeName string VLevel int VModule string + DenyPSALabel bool BindAddress string SANs []string diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 993bb2cfc591..c63a0bad3ef3 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/k3s-io/k3s/pkg/admission/denypsalabel" "github.com/k3s-io/k3s/pkg/authenticator" "github.com/k3s-io/k3s/pkg/cluster" "github.com/k3s-io/k3s/pkg/daemons/config" @@ -20,8 +21,10 @@ import ( "github.com/sirupsen/logrus" authorizationv1 "k8s.io/api/authorization/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/admission" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + "k8s.io/kubernetes/pkg/kubeapiserver/options" "k8s.io/kubernetes/pkg/registry/core/node" // for client metric registration @@ -228,6 +231,13 @@ func apiServer(ctx context.Context, cfg *config.Control) error { args := config.GetArgs(argsMap, cfg.ExtraAPIArgs) + // Add extra admission plugins + if cfg.DenyPSALabel { + extraPlugins := make(map[string]func(*admission.Plugins)) + extraPlugins[denypsalabel.PluginName] = denypsalabel.Register + options.AdmissionPlugins = extraPlugins + } + logrus.Infof("Running kube-apiserver %s", config.ArgString(args)) return executor.APIServer(ctx, runtime.ETCDReady, args) From 815f40b50525cb2d70274d3a6c401dfcc23b996d Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Thu, 26 Sep 2024 00:43:13 +0300 Subject: [PATCH 2/6] Fix comments Signed-off-by: galal-hussein --- pkg/admission/denypsalabel/admission.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/admission/denypsalabel/admission.go b/pkg/admission/denypsalabel/admission.go index 132b98556915..c45e0b807399 100644 --- a/pkg/admission/denypsalabel/admission.go +++ b/pkg/admission/denypsalabel/admission.go @@ -42,7 +42,7 @@ func Register(plugins *admission.Plugins) { }) } -// externalIPsDenierPlugin holds state for and implements the admission plugin. +// psaLabelDenialPlugin holds state for and implements the admission plugin. type psaLabelDenialPlugin struct { *admission.Handler } @@ -57,8 +57,7 @@ func newPlugin() *psaLabelDenialPlugin { } } -// Admit ensures that modifications of the Service.Spec.ExternalIPs field are -// denied +// Validate ensures that applying PSA label to namespaces is denied func (plug *psaLabelDenialPlugin) Validate(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces) error { if attr.GetResource().GroupResource() != core.Resource("namespaces") { return nil @@ -71,8 +70,8 @@ func (plug *psaLabelDenialPlugin) Validate(ctx context.Context, attr admission.A // if we can't convert then we don't handle this object so just return newNS, ok := attr.GetObject().(*core.Namespace) if !ok { - klog.V(3).Infof("Expected Namespace resource, got: %v", attr.GetKind()) - return errors.NewInternalError(fmt.Errorf("Expected Namespace resource, got: %v", attr.GetKind())) + klog.V(3).Infof("Expected namespace resource, got: %v", attr.GetKind()) + return errors.NewInternalError(fmt.Errorf("expected namespace resource, got: %v", attr.GetKind())) } if !isPSALabel(newNS) { @@ -80,7 +79,7 @@ func (plug *psaLabelDenialPlugin) Validate(ctx context.Context, attr admission.A } klog.V(4).Infof("Denying use of PSA label on namespace %s", newNS.Name) - return admission.NewForbidden(attr, fmt.Errorf("Denying use of PSA label on Namespace")) + return admission.NewForbidden(attr, fmt.Errorf("denying use of PSA label on namespace")) } func isPSALabel(newNS *core.Namespace) bool { From 369362c7349a8b8e998d6263a5d48c52c368b3bb Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Thu, 26 Sep 2024 23:43:03 +0300 Subject: [PATCH 3/6] Fix comments Signed-off-by: galal-hussein --- pkg/admission/denypsalabel/admission.go | 12 ++++++------ pkg/cli/cmds/server.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/admission/denypsalabel/admission.go b/pkg/admission/denypsalabel/admission.go index c45e0b807399..f8334c7ece49 100644 --- a/pkg/admission/denypsalabel/admission.go +++ b/pkg/admission/denypsalabel/admission.go @@ -30,8 +30,8 @@ import ( const ( // PluginName is the name of this admission controller plugin - PluginName = "DenyPSALabel" - PSALabelPrefix = "pod-security.kubernetes.io" + PluginName = "DenyPSALabel" + labelPrefix = "pod-security.kubernetes.io/" ) // Register registers a plugin @@ -71,20 +71,20 @@ func (plug *psaLabelDenialPlugin) Validate(ctx context.Context, attr admission.A newNS, ok := attr.GetObject().(*core.Namespace) if !ok { klog.V(3).Infof("Expected namespace resource, got: %v", attr.GetKind()) - return errors.NewInternalError(fmt.Errorf("expected namespace resource, got: %v", attr.GetKind())) + return errors.NewInternalError(fmt.Errorf("Expected Namespace resource, got: %v", attr.GetKind())) } if !isPSALabel(newNS) { return nil } - klog.V(4).Infof("Denying use of PSA label on namespace %s", newNS.Name) - return admission.NewForbidden(attr, fmt.Errorf("denying use of PSA label on namespace")) + klog.V(4).Infof("Denying use of label with %s prefix on Namespace %s", labelPrefix, newNS.Name) + return admission.NewForbidden(attr, fmt.Errorf("Use of label with %s prefix on Namespace is denied by admission control", labelPrefix)) } func isPSALabel(newNS *core.Namespace) bool { for labelName := range newNS.Labels { - if strings.HasPrefix(labelName, PSALabelPrefix) { + if strings.HasPrefix(labelName, labelPrefix) { return true } } diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 404ee130d6fe..92cab3ccf444 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -589,7 +589,7 @@ var ServerFlags = []cli.Flag{ }, &cli.BoolFlag{ Name: "deny-psa-label", - Usage: "(experimental) Deny modifying namespaces with PSA security label", + Usage: "(experimental) Deny use of pod-security.kubernetes.io labels on Namespaces", Hidden: true, Destination: &ServerConfig.DenyPSALabel, }, From 81261af434affaecd37adf0e4b61b3b16bf4039d Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 2 Oct 2024 23:49:26 +0300 Subject: [PATCH 4/6] Add integration test for --deny-psa-label Signed-off-by: galal-hussein --- tests/integration/startup/startup_int_test.go | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/integration/startup/startup_int_test.go b/tests/integration/startup/startup_int_test.go index e0f23f916db3..d1d4cd10efab 100644 --- a/tests/integration/startup/startup_int_test.go +++ b/tests/integration/startup/startup_int_test.go @@ -365,7 +365,28 @@ var _ = Describe("startup tests", Ordered, func() { Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) }) }) + When("a server with a --deny-psa-label is created", func() { + It("is created with no arguments", func() { + var err error + startupServerArgs = []string{ + "--deny-psa-label", + "--kube-apiserver-arg", + "enable-admission-plugins=DenyPSALabel", + } + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("change label of namespace", func() { + _, err := testutil.K3sCmd("kubectl label --dry-run=server --overwrite ns --all pod-security.kubernetes.io/enforce=baseline") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("denying use of PSA label on namespace")) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) + }) + }) }) var failed bool From f4b84b454a8e9d9d7287b9d6d967fc9480306743 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Mon, 7 Oct 2024 20:49:10 +0300 Subject: [PATCH 5/6] Fix integration test for admission plugin Signed-off-by: galal-hussein --- tests/integration/startup/startup_int_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/integration/startup/startup_int_test.go b/tests/integration/startup/startup_int_test.go index d1d4cd10efab..6452e97b5f5a 100644 --- a/tests/integration/startup/startup_int_test.go +++ b/tests/integration/startup/startup_int_test.go @@ -371,15 +371,20 @@ var _ = Describe("startup tests", Ordered, func() { startupServerArgs = []string{ "--deny-psa-label", "--kube-apiserver-arg", - "enable-admission-plugins=DenyPSALabel", + "enable-admission-plugins=\"DenyPSALabel\"", } startupServer, err = testutil.K3sStartServer(startupServerArgs...) Expect(err).ToNot(HaveOccurred()) }) + It("has the default pods deployed", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "120s", "5s").Should(Succeed()) + }) It("change label of namespace", func() { - _, err := testutil.K3sCmd("kubectl label --dry-run=server --overwrite ns --all pod-security.kubernetes.io/enforce=baseline") + res, err := testutil.K3sCmd("kubectl label --dry-run=server --overwrite ns --all pod-security.kubernetes.io/enforce=baseline") Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("denying use of PSA label on namespace")) + Expect(res).To(ContainSubstring("denying use of PSA label on namespace")) }) It("dies cleanly", func() { From 16242c62475edd8170338341e924110962d5ae8c Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Tue, 8 Oct 2024 00:42:29 +0300 Subject: [PATCH 6/6] Fix integration test for admission plugin Signed-off-by: galal-hussein --- tests/integration/startup/startup_int_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/startup/startup_int_test.go b/tests/integration/startup/startup_int_test.go index 6452e97b5f5a..b272eca4753c 100644 --- a/tests/integration/startup/startup_int_test.go +++ b/tests/integration/startup/startup_int_test.go @@ -371,7 +371,7 @@ var _ = Describe("startup tests", Ordered, func() { startupServerArgs = []string{ "--deny-psa-label", "--kube-apiserver-arg", - "enable-admission-plugins=\"DenyPSALabel\"", + "enable-admission-plugins=DenyPSALabel", } startupServer, err = testutil.K3sStartServer(startupServerArgs...) Expect(err).ToNot(HaveOccurred()) @@ -385,7 +385,6 @@ var _ = Describe("startup tests", Ordered, func() { res, err := testutil.K3sCmd("kubectl label --dry-run=server --overwrite ns --all pod-security.kubernetes.io/enforce=baseline") Expect(err).To(HaveOccurred()) Expect(res).To(ContainSubstring("denying use of PSA label on namespace")) - }) It("dies cleanly", func() { Expect(testutil.K3sKillServer(startupServer)).To(Succeed())