feat: websocket connection configuration (#3700)

* configure websocket connection Signed-off-by: ac892247 <[email protected]> * lower default values, increase in IT Signed-off-by: ac892247 <[email protected]> --------- Signed-off-by: ac892247 <[email protected]>
zowe · Aug 27, 2024 · eb98b13 · eb98b13
1 parent b1a4aa8
commit eb98b13
Show file tree

Hide file tree

Showing 10 changed files with 122 additions and 22 deletions.
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -65,6 +65,8 @@ jobs:
                     APIML_SECURITY_AUTH_PASSTICKET_CUSTOMUSERHEADER: customUserHeader
                     APIML_SECURITY_AUTH_PASSTICKET_CUSTOMAUTHHEADER: customPassticketHeader
                     ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka
+                    SERVER_MAX_HTTP_REQUEST_HEADER_SIZE: 16348
+                    SERVER_WEBSOCKET_REQUESTBUFFERSIZE: 16348
             zaas-service:
                 image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }}
                 env:

diff --git a/build.gradle b/build.gradle
@@ -230,10 +230,6 @@ task runZaasTest(dependsOn: ":integration-tests:runZaasTest") {
     group "Integration tests"
 }
 
-task runBaseTestsInternalPort(dependsOn: ":integration-tests:runBaseTestsInternalPort") {
-    description "Run base tests on internal port"
-    group "Integration tests"
-}
 task runCachingServiceTests(dependsOn: ":integration-tests:runCachingServiceTests") {
     description "Run caching service tests"
     group "Integration tests"

diff --git a/discoverable-client/src/main/resources/application.yml b/discoverable-client/src/main/resources/application.yml
@@ -12,6 +12,7 @@ logging:
         com.netflix.discovery.DiscoveryClient: OFF
         org.springframework.boot.web.embedded.tomcat.TomcatWebServer: INFO
         org.apache.http.conn.ssl.DefaultHostnameVerifier: DEBUG #logs only SSLException
+        org.springframework.web.socket.adapter: DEBUG
 
 spring:
     cloud:
@@ -160,7 +161,7 @@ server:
         mime-types: image/png
         enabled: true
         min-response-size: 500
-
+    max-http-request-header-size: 65536
 management:
     endpoints:
         migrate-legacy-ids: true

diff --git a/gateway-package/src/main/resources/bin/start.sh b/gateway-package/src/main/resources/bin/start.sh
@@ -216,6 +216,10 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${GATEWAY_CODE} java \
     -Dserver.ssl.trustStore="${truststore_location}" \
     -Dserver.ssl.trustStoreType="${truststore_type}" \
     -Dserver.ssl.trustStorePassword="${truststore_pass}" \
+    -Dserver.webSocket.maxIdleTimeout=${ZWE_configs_server_webSocket_maxIdleTimeout:-3600000} \
+    -Dserver.webSocket.connectTimeout=${ZWE_configs_server_webSocket_connectTimeout:-45000} \
+    -Dserver.webSocket.asyncWriteTimeout=${ZWE_configs_server_webSocket_asyncWriteTimeout:-60000} \
+    -Dserver.webSocket.requestBufferSize=${ZWE_configs_server_webSocket_requestBufferSize:-8192} \
     -Djava.protocol.handler.pkgs=com.ibm.crypto.provider \
     -Djavax.net.debug=${ZWE_configs_sslDebug:-""} \
     -Djava.library.path=${LIBPATH} \

diff --git a/gateway-service/src/main/java/org/zowe/apiml/gateway/config/WebSocketConfig.java b/gateway-service/src/main/java/org/zowe/apiml/gateway/config/WebSocketConfig.java
@@ -10,6 +10,8 @@
 
 package org.zowe.apiml.gateway.config;
 
+import org.apache.tomcat.websocket.WsWebSocketContainer;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Primary;
@@ -21,10 +23,22 @@
 @Configuration
 public class WebSocketConfig {
 
+    @Value("${server.webSocket.requestBufferSize:8192}")
+    private int bufferSize;
+    @Value("${server.webSocket.asyncWriteTimeout:60000}")
+    private long sendTimeout;
+    @Value("${server.webSocket.maxIdleTimeout:3600000}")
+    private long idleTimeout;
+
     @Bean
     @Primary
     public WebSocketClient tomcatWebSocketClient() {
-        return new ApimlWebSocketClient();
+        var wsContainer = new WsWebSocketContainer();
+        wsContainer.setDefaultMaxTextMessageBufferSize(bufferSize);
+        wsContainer.setDefaultMaxBinaryMessageBufferSize(bufferSize);
+        wsContainer.setAsyncSendTimeout(sendTimeout);
+        wsContainer.setDefaultMaxSessionIdleTimeout(idleTimeout);
+        return new ApimlWebSocketClient(wsContainer);
     }
 
     @Bean

diff --git a/gateway-service/src/main/java/org/zowe/apiml/gateway/websocket/ApimlWebSocketClient.java b/gateway-service/src/main/java/org/zowe/apiml/gateway/websocket/ApimlWebSocketClient.java
@@ -10,23 +10,43 @@
 
 package org.zowe.apiml.gateway.websocket;
 
+import jakarta.websocket.ClientEndpointConfig;
 import jakarta.websocket.Session;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.tomcat.websocket.Constants;
 import org.apache.tomcat.websocket.WsWebSocketContainer;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.web.reactive.socket.HandshakeInfo;
 import org.springframework.web.reactive.socket.adapter.StandardWebSocketSession;
 import org.springframework.web.reactive.socket.client.StandardWebSocketClient;
 import reactor.core.publisher.Sinks;
 
+import java.util.List;
+
+@Slf4j
 public class ApimlWebSocketClient extends StandardWebSocketClient {
 
-    public ApimlWebSocketClient() {
-        super(new WsWebSocketContainer());
+    @Value("${server.webSocket.connectTimeout:45000}")
+    private String ioTimeout = "45000";
+
+    public ApimlWebSocketClient(WsWebSocketContainer webSocketContainer) {
+        super(webSocketContainer);
     }
 
     @Override
     protected StandardWebSocketSession createWebSocketSession(
         Session session, HandshakeInfo info, Sinks.Empty<Void> completionSink) {
-
         return new ApimlWebSocketSession(session, info, bufferFactory(), completionSink);
     }
+
+    @Override
+    protected ClientEndpointConfig createEndpointConfig(ClientEndpointConfig.Configurator configurator, List<String> subProtocols) {
+        var config = ClientEndpointConfig.Builder.create()
+            .configurator(configurator)
+            .preferredSubprotocols(subProtocols)
+            .build();
+        config.getUserProperties().put(Constants.IO_TIMEOUT_MS_PROPERTY, ioTimeout);
+        return config;
+    }
+
 }
diff --git a/gateway-service/src/main/resources/application.yml b/gateway-service/src/main/resources/application.yml
@@ -153,10 +153,12 @@ logging:
         org.springframework.cloud.gateway: DEBUG
         org.springframework.security: DEBUG
         org.springframework.web.reactive: DEBUG
+        org.springframework.web.reactive.socket: DEBUG
         reactor.netty.http.client: DEBUG
         reactor.netty.http.client.HttpClient: DEBUG
         reactor.netty.http.client.HttpClientConnect: DEBUG
         com.netflix: DEBUG
+        org.apache.tomcat.util.net: DEBUG
 
 ---
 spring.config.activate.on-profile: attls

diff --git a/gateway-service/src/test/java/org/zowe/apiml/gateway/websocket/ApimlWebSocketClientTest.java b/gateway-service/src/test/java/org/zowe/apiml/gateway/websocket/ApimlWebSocketClientTest.java
@@ -0,0 +1,33 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+package org.zowe.apiml.gateway.websocket;
+
+import jakarta.websocket.ClientEndpointConfig;
+import org.apache.tomcat.websocket.Constants;
+import org.apache.tomcat.websocket.WsWebSocketContainer;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+class ApimlWebSocketClientTest {
+
+    @Test
+    void whenCreateEndpointConfig_thenConfigContainsUserProperties() {
+        var wsContainer = mock(WsWebSocketContainer.class);
+        var client = new ApimlWebSocketClient(wsContainer);
+        var configurator = mock(ClientEndpointConfig.Configurator.class);
+        var endpointConfig = client.createEndpointConfig(configurator, Collections.emptyList());
+        assertTrue(endpointConfig.getUserProperties().containsKey(Constants.IO_TIMEOUT_MS_PROPERTY));
+    }
+}
diff --git a/integration-tests/build.gradle b/integration-tests/build.gradle
@@ -303,6 +303,7 @@ task runContainerTests(type: Test) {
 task runBaseTests(type: Test) {
     group "integration tests"
     description "Run base tests"
+    systemProperty "environment.offPlatform", "true"
 
     outputs.cacheIf { false }
 
@@ -315,8 +316,7 @@ task runBaseTests(type: Test) {
             'WebsocketTest',
             'GeneralAuthenticationTest',
             'DiscoverableClientDependentTest',
-            'CachingServiceTest',
-            'WebsocketTest'
+            'CachingServiceTest'
         )
         excludeTags(
             'MainframeDependentTests',
@@ -481,17 +481,6 @@ task runInfinispanServiceTests(type: Test) {
     }
 }
 
-task runBaseTestsInternalPort(type: Test) {
-    group "Integration tests"
-    description "Run only tests without long tests"
-
-    outputs.cacheIf { false }
-
-    dependsOn runBaseTests
-    systemProperties System.properties
-    systemProperty "gateway.port", System.getProperty("internal.gateway.port")
-}
-
 task runHATests(type: Test) {
     group "Integration tests"
     description "Run tests verifying High Availability"

diff --git a/integration-tests/src/test/java/org/zowe/apiml/integration/proxy/WebSocketProxyTest.java b/integration-tests/src/test/java/org/zowe/apiml/integration/proxy/WebSocketProxyTest.java
@@ -11,6 +11,7 @@
 package org.zowe.apiml.integration.proxy;
 
 import io.restassured.RestAssured;
+import jakarta.websocket.ContainerProvider;
 import org.apache.http.client.utils.URIBuilder;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
@@ -32,6 +33,7 @@
 
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Arrays;
 import java.util.Base64;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -90,6 +92,11 @@ public void afterConnectionClosed(WebSocketSession session, CloseStatus status)
                     }
                 }
             }
+
+            @Override
+            public boolean supportsPartialMessages() {
+                return true;
+            }
         };
     }
 
@@ -103,7 +110,13 @@ private String discoverableClientGatewayUrl(String gatewayUrl) throws URISyntaxE
 
     private WebSocketSession appendingWebSocketSession(String url, WebSocketHttpHeaders headers, StringBuilder response, int countToNotify)
         throws Exception {
+        var wsContainer = ContainerProvider.getWebSocketContainer();
+        wsContainer.setDefaultMaxTextMessageBufferSize(Integer.MAX_VALUE);
+        wsContainer.setDefaultMaxBinaryMessageBufferSize(Integer.MAX_VALUE);
+        wsContainer.setAsyncSendTimeout(10_000_000L);
+        wsContainer.setDefaultMaxSessionIdleTimeout(10_000_000L);
         StandardWebSocketClient client = new StandardWebSocketClient();
+
         client.setSslContext(HttpClientUtils.ignoreSslContext());
         URI uri = UriComponentsBuilder.fromUriString(url).build().encode().toUri();
         return client.execute(appendResponseHandler(response, countToNotify), headers, uri).get(30000, TimeUnit.MILLISECONDS);
@@ -187,6 +200,32 @@ void whenUrlFormatIsNotCorrect() throws Exception {
                         assertEquals("jakarta.websocket.DeploymentException: The HTTP response from the server [404] did not permit the HTTP upgrade to WebSocket",
                             exception.getMessage());
                     }
+
+                    @Test
+                    void whenHandshakeRequestIsTooLarge() throws Exception {
+                        final StringBuilder response = new StringBuilder();
+                        if (!VALID_AUTH_HEADERS.containsKey("X-Test")) {
+                            VALID_AUTH_HEADERS.add("X-Test", "value");
+                        }
+                        VALID_AUTH_HEADERS.add("Cookie", validToken);
+
+                        char[] data = new char[3000];
+                        Arrays.fill(data, 'a');
+                        for (int i = 0; i < 3; i++) {
+                            VALID_AUTH_HEADERS.add("X-Test-" + i, String.valueOf(data));
+                        }
+                        WebSocketSession session = appendingWebSocketSession(discoverableClientGatewayUrl(DISCOVERABLE_WS_HEADER), VALID_AUTH_HEADERS, response, 1);
+
+                        session.sendMessage(new TextMessage("gimme those headers"));
+                        synchronized (response) {
+                            response.wait();
+                        }
+
+
+                        assertTrue(response.toString().contains("x-test-2"), "WebSocket response: " + response + ". Does not contain x-test-2");
+                        session.close();
+
+                    }
                 }
             }