introduces Enhanced Host Monitoring v.2 plugin (#513)

performance tests
fixes and docs

Co-authored-by: sergiyv-bitquill <[email protected]>

README.md

@@ -529,6 +529,26 @@ You can include additional monitoring configurations by adding the prefix `monit
 > 
 > It is suggested to turn off Enhanced Failure Monitoring plugin, or to avoid using RDS Proxy endpoints when the plugin is active. 
 
+## **Experimental** Enhanced Failure Monitoring Plugin v2
+
+> [!WARNING] 
+> This plugin is experimental and users should test the plugin before using it in production environment.
+
+Enhanced Failure Monitoring Plugin v2 is an alternative implementation of enhanced failure monitoring, and it is functionally equal to the Enhanced Failure Monitoring Plugin described above. Both plugins share the same set of [configuration parameters](#enhanced-failure-monitoring-parameters). Enhanced Failure Monitoring Plugin v2 plugin is designed to be a drop-in replacement for the Enhanced Failure Monitoring Plugin.
+
+> [!NOTE] 
+> Since these two plugins are separate plugins, users may decide to use them together with a single connection. While this should not  have any negative side effects, it is not recommended. It is recommended to use either the Enhanced Failure Monitoring Plugin, or the Enhanced Failure Monitoring Plugin v2 where it's needed.
+
+In order to use Enhanced Failure Monitoring Plugin v2, users should add `software.aws.rds.jdbc.mysql.shading.com.mysql.cj.jdbc.ha.plugins.efm2.NodeMonitoringConnectionPluginFactory` to `connectionPluginFactories`.
+
+Enhanced Failure Monitoring Plugin v2 is designed to address [some of the issues](https://github.com/awslabs/aws-mysql-jdbc/issues/412) that have been reported by multiple users. The following changes have been made:
+- Used weak pointers to ease garbage collection
+- Split monitoring logic into two separate threads to increase overall monitoring stability
+- Reviewed locks for monitoring context
+- Reviewed and redesigned stopping of idle monitoring threads
+- Reviewed and simplified monitoring logic
+
+
 ## AWS Secrets Manager Plugin
 
 The AWS JDBC Driver for MySQL supports usage of database credentials stored in the [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/) through the AWS Secrets Manager Plugin. This plugin is optional and can be enabled with the `connectionPluginFactories` parameter as seen in the [connection plugin manager parameters table](#connection-plugin-manager-parameters). When a user creates a new connection with this plugin enabled, the plugin will retrieve the secret and the connection will be created using those credentials.

build.gradle.kts

@@ -136,7 +136,7 @@ tasks.shadowJar {
     exclude("instrumentation/**")
     exclude("demo/**")
     exclude("documentation/**")
-    exclude("customplugins/**");
+    exclude("customplugins/**")
 
     includeEmptyDirs = false
 }
@@ -447,7 +447,10 @@ tasks.register<Test>("in-container-aurora") {
 }
 
 tasks.register<Test>("in-container-aurora-performance") {
-    filter.includeTestsMatching("testsuite.integration.container.AuroraMysqlPerformanceIntegrationTest")
+    filter.includeTestsMatching(
+        "testsuite.integration.container.AuroraMysqlPerformanceIntegrationTest")
+    filter.includeTestsMatching(
+        "testsuite.integration.container.AuroraMysqlPerformanceForEfm2IntegrationTest")
 }
 
 // Run all tests excluding integration tests

src/main/core-api/java/com/mysql/cj/conf/PropertyDefinitions.java

@@ -716,7 +716,7 @@ public enum DatabaseTerm {
                         Messages.getString("ConnectionProperties.failureDetectionCount"), "0.4.0", CATEGORY_HA, Integer.MAX_VALUE, 0,
                         Integer.MAX_VALUE),
 
-                new IntegerPropertyDefinition(PropertyKey.monitorDisposalTime, 60_000, RUNTIME_MODIFIABLE,
+                new IntegerPropertyDefinition(PropertyKey.monitorDisposalTime, 900_000, RUNTIME_MODIFIABLE, // 15 min
                     Messages.getString("ConnectionProperties.monitorDisposalTime"), "0.4.0", CATEGORY_HA, Integer.MAX_VALUE, 0,
                     Integer.MAX_VALUE),
 

src/main/user-impl/java/com/mysql/cj/jdbc/ha/plugins/efm2/DefaultMonitorService.java

@@ -0,0 +1,174 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2.0
+ * (GPLv2), as published by the Free Software Foundation, with the
+ * following additional permissions:
+ *
+ * This program is distributed with certain software that is licensed
+ * under separate terms, as designated in a particular file or component
+ * or in the license documentation. Without limiting your rights under
+ * the GPLv2, the authors of this program hereby grant you an additional
+ * permission to link the program and your derivative works with the
+ * separately licensed software that they have included with the program.
+ *
+ * Without limiting the foregoing grant of rights under the GPLv2 and
+ * additional permission as to separately licensed software, this
+ * program is also subject to the Universal FOSS Exception, version 1.0,
+ * a copy of which can be found along with its FAQ at
+ * http://oss.oracle.com/licenses/universal-foss-exception.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License, version 2.0, for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ */
+
+package com.mysql.cj.jdbc.ha.plugins.efm2;
+
+import com.mysql.cj.conf.HostInfo;
+import com.mysql.cj.conf.PropertyKey;
+import com.mysql.cj.conf.PropertySet;
+import com.mysql.cj.jdbc.JdbcConnection;
+import com.mysql.cj.jdbc.ha.plugins.BasicConnectionProvider;
+import com.mysql.cj.jdbc.ha.util.SlidingExpirationCacheWithCleanupThread;
+import com.mysql.cj.log.Log;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import org.checkerframework.checker.nullness.qual.NonNull;
+
+/**
+ * This class handles the creation and clean up of monitoring threads to servers with one
+ * or more active connections.
+ */
+public class DefaultMonitorService implements IMonitorService {
+  protected static final long CACHE_CLEANUP_NANO = TimeUnit.MINUTES.toNanos(1);
+
+  protected static final Executor ABORT_EXECUTOR = Executors.newSingleThreadExecutor();
+
+  protected static final SlidingExpirationCacheWithCleanupThread<String, IMonitor> monitors =
+      new SlidingExpirationCacheWithCleanupThread<>(
+          IMonitor::canDispose,
+          (monitor) -> {
+            try {
+              monitor.close();
+            } catch (Exception ex) {
+              // ignore
+            }
+          },
+          CACHE_CLEANUP_NANO);
+
+  protected final Log logger;
+  protected final IMonitorInitializer monitorInitializer;
+
+  public DefaultMonitorService(Log logger) {
+    this(
+        (hostInfo,
+            propertySet,
+            failureDetectionTimeMillis,
+            failureDetectionIntervalMillis,
+            failureDetectionCount) ->
+            new Monitor(
+              new BasicConnectionProvider(),
+              hostInfo,
+              propertySet,
+              failureDetectionTimeMillis,
+              failureDetectionIntervalMillis,
+              failureDetectionCount,
+              logger),
+        logger
+    );
+  }
+
+  DefaultMonitorService(IMonitorInitializer monitorInitializer, Log logger) {
+    this.monitorInitializer = monitorInitializer;
+    this.logger = logger;
+  }
+
+  @Override
+  public MonitorConnectionContext startMonitoring(
+      JdbcConnection connectionToAbort,
+      HostInfo hostInfo,
+      PropertySet propertySet,
+      int failureDetectionTimeMillis,
+      int failureDetectionIntervalMillis,
+      int failureDetectionCount) {
+
+    final IMonitor monitor = this.getMonitor(
+        hostInfo,
+        propertySet,
+        failureDetectionTimeMillis,
+        failureDetectionIntervalMillis,
+        failureDetectionCount);
+
+    final MonitorConnectionContext context = new MonitorConnectionContext(connectionToAbort);
+    monitor.startMonitoring(context);
+
+    return context;
+  }
+
+  @Override
+  public void stopMonitoring(
+      @NonNull final MonitorConnectionContext context,
+      @NonNull Connection connectionToAbort) {
+
+    if (context.shouldAbort()) {
+      context.setInactive();
+      try {
+        connectionToAbort.abort(ABORT_EXECUTOR);
+        connectionToAbort.close();
+      } catch (final SQLException sqlEx) {
+        // ignore
+        if (logger.isTraceEnabled()) {
+          logger.logTrace(
+              String.format(
+                  "[efm2.DefaultMonitorService.stopMonitoring]: Exception during aborting connection: %s",
+                  sqlEx.getMessage()));
+        }
+      }
+    } else {
+      context.setInactive();
+    }
+  }
+
+  @Override
+  public void releaseResources() {
+    // do nothing
+  }
+
+  protected IMonitor getMonitor(
+      HostInfo hostInfo,
+      PropertySet propertySet,
+      final int failureDetectionTimeMillis,
+      final int failureDetectionIntervalMillis,
+      final int failureDetectionCount) {
+
+    final String monitorKey = String.format("%d:%d:%d:%s",
+        failureDetectionTimeMillis,
+        failureDetectionIntervalMillis,
+        failureDetectionCount,
+        hostInfo.getHostPortPair());
+
+    final long cacheExpirationNano = TimeUnit.MILLISECONDS.toNanos(
+        propertySet.getIntegerProperty(PropertyKey.monitorDisposalTime).getValue());
+
+    return monitors.computeIfAbsent(
+        monitorKey,
+        (key) -> monitorInitializer.createMonitor(
+            hostInfo,
+            propertySet,
+            failureDetectionTimeMillis,
+            failureDetectionIntervalMillis,
+            failureDetectionCount),
+        cacheExpirationNano);
+  }
+}

src/main/user-impl/java/com/mysql/cj/jdbc/ha/plugins/efm2/IMonitor.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2.0
+ * (GPLv2), as published by the Free Software Foundation, with the
+ * following additional permissions:
+ *
+ * This program is distributed with certain software that is licensed
+ * under separate terms, as designated in a particular file or component
+ * or in the license documentation. Without limiting your rights under
+ * the GPLv2, the authors of this program hereby grant you an additional
+ * permission to link the program and your derivative works with the
+ * separately licensed software that they have included with the program.
+ *
+ * Without limiting the foregoing grant of rights under the GPLv2 and
+ * additional permission as to separately licensed software, this
+ * program is also subject to the Universal FOSS Exception, version 1.0,
+ * a copy of which can be found along with its FAQ at
+ * http://oss.oracle.com/licenses/universal-foss-exception.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License, version 2.0, for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ */
+
+package com.mysql.cj.jdbc.ha.plugins.efm2;
+
+/**
+ * Interface for monitors. This class uses background threads to monitor servers with one
+ * or more connections for more efficient failure detection during method execution.
+ */
+public interface IMonitor extends AutoCloseable, Runnable {
+  void startMonitoring(MonitorConnectionContext context);
+
+  boolean canDispose();
+}

src/main/user-impl/java/com/mysql/cj/jdbc/ha/plugins/efm2/IMonitorInitializer.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2.0
+ * (GPLv2), as published by the Free Software Foundation, with the
+ * following additional permissions:
+ *
+ * This program is distributed with certain software that is licensed
+ * under separate terms, as designated in a particular file or component
+ * or in the license documentation. Without limiting your rights under
+ * the GPLv2, the authors of this program hereby grant you an additional
+ * permission to link the program and your derivative works with the
+ * separately licensed software that they have included with the program.
+ *
+ * Without limiting the foregoing grant of rights under the GPLv2 and
+ * additional permission as to separately licensed software, this
+ * program is also subject to the Universal FOSS Exception, version 1.0,
+ * a copy of which can be found along with its FAQ at
+ * http://oss.oracle.com/licenses/universal-foss-exception.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License, version 2.0, for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ */
+
+package com.mysql.cj.jdbc.ha.plugins.efm2;
+
+import com.mysql.cj.conf.HostInfo;
+import com.mysql.cj.conf.PropertySet;
+import com.mysql.cj.log.Log;
+
+/**
+ * Interface for initialize a new {@link Monitor}.
+ */
+@FunctionalInterface
+public interface IMonitorInitializer {
+  IMonitor createMonitor(
+      HostInfo hostInfo,
+      PropertySet propertySet,
+      final int failureDetectionTimeMillis,
+      final int failureDetectionIntervalMillis,
+      final int failureDetectionCount);
+}

src/main/user-impl/java/com/mysql/cj/jdbc/ha/plugins/efm2/IMonitorService.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2.0
+ * (GPLv2), as published by the Free Software Foundation, with the
+ * following additional permissions:
+ *
+ * This program is distributed with certain software that is licensed
+ * under separate terms, as designated in a particular file or component
+ * or in the license documentation. Without limiting your rights under
+ * the GPLv2, the authors of this program hereby grant you an additional
+ * permission to link the program and your derivative works with the
+ * separately licensed software that they have included with the program.
+ *
+ * Without limiting the foregoing grant of rights under the GPLv2 and
+ * additional permission as to separately licensed software, this
+ * program is also subject to the Universal FOSS Exception, version 1.0,
+ * a copy of which can be found along with its FAQ at
+ * http://oss.oracle.com/licenses/universal-foss-exception.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License, version 2.0, for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ */
+
+package com.mysql.cj.jdbc.ha.plugins.efm2;
+
+import com.mysql.cj.conf.HostInfo;
+import com.mysql.cj.conf.PropertySet;
+import com.mysql.cj.jdbc.JdbcConnection;
+
+import java.sql.Connection;
+
+/**
+ * Interface for monitor services. This class implements ways to start and stop monitoring
+ * servers when connections are created.
+ */
+public interface IMonitorService {
+  MonitorConnectionContext startMonitoring(
+      JdbcConnection connectionToAbort,
+      HostInfo hostInfo,
+      PropertySet propertySet,
+      int failureDetectionTimeMillis,
+      int failureDetectionIntervalMillis,
+      int failureDetectionCount);
+
+  /**
+   * Stop monitoring for a connection represented by the given {@link MonitorConnectionContext}.
+   * Removes the context from the {@link IMonitor}.
+   *
+   * @param context The {@link MonitorConnectionContext} representing a connection.
+   */
+  void stopMonitoring(MonitorConnectionContext context, Connection connectionToAbort);
+
+  void releaseResources();
+}