Adds support for MultiProcessValue ids based on env variable (#760)

* Adds support for `MultiProcessValue` ids based on env variable When baseplate is running as part of Monoceros process restarts create too many files which make prometheus client extremely slow and eventually it timeouts. We want to be able to provide an alternative id to prometheus client now based on an environment variable. To achieve this we set the process id to: * `MULTIPROCESS_WORKER_ID` if provided and fallback to PID in case it doesn't exist * Moved the prometheus client initialization before we make the server. Signed-off-by: Sotiris Nanopoulos <[email protected]> * fix liniting Signed-off-by: Sotiris Nanopoulos <[email protected]> * fix lint issues Signed-off-by: Sotiris Nanopoulos <[email protected]> * added a comment Signed-off-by: Sotiris Nanopoulos <[email protected]> Signed-off-by: Sotiris Nanopoulos <[email protected]>
reddit · Nov 30, 2022 · 2b53adc · 2b53adc
1 parent 3ebe86c
commit 2b53adc
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 8 deletions.
diff --git a/baseplate/server/__init__.py b/baseplate/server/__init__.py
@@ -265,6 +265,15 @@ def load_app_and_run_server() -> None:
         config = read_config(args.config_file, args.server_name, args.app_name)
     assert config.server
 
+    # Prom exporter needs to start before the app starts because
+    # we need to set the prometheus id before we generate any stats
+    if is_metrics_enabled(config.app):
+        from baseplate.server.prometheus import start_prometheus_exporter
+
+        start_prometheus_exporter()
+    else:
+        logger.info("Metrics are not configured, Prometheus metrics will not be exported.")
+
     configure_logging(config, args.debug)
 
     app = make_app(config.app)
@@ -274,13 +283,6 @@ def load_app_and_run_server() -> None:
     if einhorn.is_worker():
         einhorn.ack_startup()
 
-    if is_metrics_enabled(config.app):
-        from baseplate.server.prometheus import start_prometheus_exporter
-
-        start_prometheus_exporter()
-    else:
-        logger.info("Metrics are not configured, Prometheus metrics will not be exported.")
-
     if args.reload:
         reloader.start_reload_watcher(extra_files=[args.config_file.name])
 

diff --git a/baseplate/server/prometheus.py b/baseplate/server/prometheus.py
@@ -24,6 +24,8 @@
 from prometheus_client import CONTENT_TYPE_LATEST
 from prometheus_client import generate_latest
 from prometheus_client import multiprocess
+from prometheus_client import values
+from prometheus_client.values import MultiProcessValue
 
 from baseplate.lib.config import Endpoint
 from baseplate.lib.config import EndpointConfiguration
@@ -40,6 +42,13 @@
 METRICS_ENDPOINT = "/metrics"
 
 
+def worker_id() -> str:
+    worker = os.environ.get("MULTIPROCESS_WORKER_ID")
+    if worker is None:
+        worker = str(os.getpid())
+    return worker
+
+
 def export_metrics(environ: "WSGIEnvironment", start_response: "StartResponse") -> Iterable[bytes]:
     if environ["PATH_INFO"] != METRICS_ENDPOINT:
         start_response("404 Not Found", [("Content-Type", "text/plain")])
@@ -60,7 +69,8 @@ def start_prometheus_exporter(address: EndpointConfiguration = PROMETHEUS_EXPORT
         )
         sys.exit(1)
 
-    atexit.register(multiprocess.mark_process_dead, os.getpid())
+    values.ValueClass = MultiProcessValue(worker_id)
+    atexit.register(multiprocess.mark_process_dead, worker_id())
 
     server_socket = bind_socket(address)
     server = WSGIServer(