diff --git a/transport/internet/system_listener.go b/transport/internet/system_listener.go
index 6541e95f7456..1d6358978925 100644
--- a/transport/internet/system_listener.go
+++ b/transport/internet/system_listener.go
@@ -57,6 +57,10 @@ func getControlFunc(ctx context.Context, sockopt *SocketConfig, controllers []co
 func (dl *DefaultListener) Listen(ctx context.Context, addr net.Addr, sockopt *SocketConfig) (l net.Listener, err error) {
 	var lc net.ListenConfig
 	var network, address string
+	// callback is called after the Listen function returns
+	callback := func(l net.Listener, err error) (net.Listener, error) {
+		return l, err
+	}
 
 	switch addr := addr.(type) {
 	case *net.TCPAddr:
@@ -71,23 +75,6 @@ func (dl *DefaultListener) Listen(ctx context.Context, addr net.Addr, sockopt *S
 		network = addr.Network()
 		address = addr.Name
 
-		if s := strings.Split(address, ","); len(s) == 2 {
-			address = s[0]
-			perm, perr := strconv.ParseUint(s[1], 8, 32)
-			if perr != nil {
-				return nil, newError("failed to parse permission: " + s[1]).Base(perr)
-			}
-
-			defer func(file string, permission os.FileMode) {
-				if err == nil {
-					cerr := os.Chmod(address, permission)
-					if cerr != nil {
-						err = newError("failed to set permission for " + file).Base(cerr)
-					}
-				}
-			}(address, os.FileMode(perm))
-		}
-
 		if (runtime.GOOS == "linux" || runtime.GOOS == "android") && address[0] == '@' {
 			// linux abstract unix domain socket is lockfree
 			if len(address) > 1 && address[1] == '@' {
@@ -97,6 +84,18 @@ func (dl *DefaultListener) Listen(ctx context.Context, addr net.Addr, sockopt *S
 				address = string(fullAddr)
 			}
 		} else {
+			// split permission from address
+			var filePerm *os.FileMode
+			if s := strings.Split(address, ","); len(s) == 2 {
+				address = s[0]
+				perm, perr := strconv.ParseUint(s[1], 8, 32)
+				if perr != nil {
+					return nil, newError("failed to parse permission: " + s[1]).Base(perr)
+				}
+
+				mode := os.FileMode(perm)
+				filePerm = &mode
+			}
 			// normal unix domain socket needs lock
 			locker := &FileLocker{
 				path: address + ".lock",
@@ -104,18 +103,29 @@ func (dl *DefaultListener) Listen(ctx context.Context, addr net.Addr, sockopt *S
 			if err := locker.Acquire(); err != nil {
 				return nil, err
 			}
-			// combine listener and unix domain socket locker
-			defer func(locker *FileLocker) {
-				if err == nil {
-					l = &combinedListener{Listener: l, locker: locker}
-				} else {
+
+			// set callback to combine listener and set permission
+			callback = func(l net.Listener, err error) (net.Listener, error) {
+				if err != nil {
 					locker.Release()
+					return l, err
 				}
-			}(locker)
+				l = &combinedListener{Listener: l, locker: locker}
+				if filePerm == nil {
+					return l, nil
+				}
+				err = os.Chmod(address, *filePerm)
+				if err != nil {
+					l.Close()
+					return nil, newError("failed to set permission for " + address).Base(err)
+				}
+				return l, nil
+			}
 		}
 	}
 
 	l, err = lc.Listen(ctx, network, address)
+	l, err = callback(l, err)
 	if sockopt != nil && sockopt.AcceptProxyProtocol {
 		policyFunc := func(upstream net.Addr) (proxyproto.Policy, error) { return proxyproto.REQUIRE, nil }
 		l = &proxyproto.Listener{Listener: l, Policy: policyFunc}