diff --git a/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go b/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go
index 549c1c81..1131f070 100644
--- a/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go
+++ b/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go
@@ -1348,6 +1348,19 @@ func (r *Reconciler) deleteSecGroups(
 	osClient openstack.Client,
 	lb *yawolv1beta1.LoadBalancer,
 ) (bool, error) {
+	// skip deletion and release status when annotated
+	if keep, err := strconv.ParseBool(lb.GetAnnotations()[yawolv1beta1.LoadBalancerKeepSecurityGroup]); err == nil && keep {
+		if lb.Status.SecurityGroupID == nil {
+			return false, nil
+		}
+		r.Log.Info("security group was released", "lb", lb.Namespace+"/"+lb.Name)
+		err = helper.RemoveFromLBStatus(ctx, r.Client.Status(), lb, "security_group_id")
+		if err != nil {
+			return true, fmt.Errorf("failed to remove from lb status: %w", err)
+		}
+		return false, nil
+	}
+
 	var err error
 
 	portClient, err := osClient.PortClient(ctx)
@@ -1364,18 +1377,6 @@ func (r *Reconciler) deleteSecGroups(
 	if err != nil {
 		return false, fmt.Errorf("failed to delete sec group usages: %w", err)
 	}
-	// skip deletion and release status when annotated
-	if keep, err := strconv.ParseBool(lb.GetAnnotations()[yawolv1beta1.LoadBalancerKeepSecurityGroup]); err == nil && keep {
-		if lb.Status.SecurityGroupID == nil {
-			return false, nil
-		}
-		r.Log.Info("security group was released", "lb", lb.Namespace+"/"+lb.Name)
-		err = helper.RemoveFromLBStatus(ctx, r.Client.Status(), lb, "security_group_id")
-		if err != nil {
-			return true, fmt.Errorf("failed to remove from lb status: %w", err)
-		}
-		return false, nil
-	}
 
 	var requeue bool
 	if lb.Status.SecurityGroupID != nil {
diff --git a/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go b/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go
index f4b807e8..365a68b7 100644
--- a/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go
+++ b/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go
@@ -10,6 +10,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gstruct"
 
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips"
@@ -811,13 +812,24 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() {
 				}, timeout, interval).Should(Succeed())
 			})
 			It("should not delete the security group", func() {
+				portClient := mockClient.PortClientObj
+				auxiliaryPortName := "auxiliary-port"
+				securityGroupID := ""
+
 				By("checking that secgroup is set")
 				hopefully(nn, func(g Gomega, act LB) error {
 					g.Expect(act.Status.SecurityGroupID).To(Not(BeNil()))
 					g.Expect(*act.Status.SecurityGroupName == nn.String())
+					securityGroupID = *act.Status.SecurityGroupID
 					return nil
 				})
 
+				_, err := portClient.Create(ctx, ports.CreateOpts{
+					Name:           auxiliaryPortName,
+					SecurityGroups: &[]string{securityGroupID},
+				})
+				Expect(err).NotTo(HaveOccurred())
+
 				By("deleting the LB")
 				cleanupLB(nn, timeout)
 
@@ -829,6 +841,15 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() {
 					g.Expect(err).To(Not(HaveOccurred()))
 					g.Expect(len(groups)).To(Equal(1))
 				}, timeout, interval).Should(Succeed())
+
+				By("checking the security group assignment is still there")
+				portList, err := portClient.List(ctx, ports.ListOpts{})
+				Expect(err).NotTo(HaveOccurred())
+				port := ports.Port{}
+				Expect(portList).To(ContainElement(MatchFields(IgnoreExtras, Fields{
+					"Name": Equal(auxiliaryPortName),
+				}), &port))
+				Expect(port.SecurityGroups).To(ConsistOf(securityGroupID))
 			})
 			It("should not delete the fip", func() {
 				var fipIP *string