docker: also expose Prometheus metrics for Kafka

docker/docker-compose-monitoring.yml

@@ -25,6 +25,17 @@ services:
       - traefik.http.routers.prometheus.rule=PathPrefix(`/prometheus`)
       - traefik.http.services.prometheus.loadbalancer.server.port=9090
 
+  kafka-jmx-exporter:
+    image: bitnami/jmx-exporter:0
+    restart: unless-stopped
+    depends_on:
+      - kafka
+    volumes:
+      - ./kafka-jmx-exporter-config.yaml:/etc/jmx-kafka/config.yaml:ro
+    command:
+      - "5556"
+      - /etc/jmx-kafka/config.yaml
+
   kafka-ui:
     image: provectuslabs/kafka-ui:v0.5.0
     restart: unless-stopped

docker/docker-compose.yml

@@ -40,6 +40,7 @@ services:
       - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT
       - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092
       - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+      - JMX_PORT=5555
     restart: unless-stopped
     depends_on:
       - zookeeper

docker/kafka-jmx-exporter-config.yaml

@@ -0,0 +1,61 @@
+---
+jmxUrl: service:jmx:rmi:///jndi/rmi://kafka:5555/jmxrmi
+lowercaseOutputName: true
+lowercaseOutputLabelNames: true
+ssl: false
+
+# This is mostly stolen from Bitnami Helm chart for Kafka.
+# https://github.com/bitnami/charts/blob/main/bitnami/kafka/templates/metrics/jmx-configmap.yaml
+whitelistObjectNames:
+  - kafka.controller:*
+  - kafka.server:*
+  - java.lang:*
+  - kafka.network:*
+  - kafka.log:*
+rules:
+  - pattern: kafka.controller<type=(ControllerChannelManager), name=(QueueSize), broker-id=(\d+)><>(Value)
+    name: kafka_controller_$1_$2_$4
+    labels:
+      broker_id: "$3"
+  - pattern: kafka.controller<type=(ControllerChannelManager), name=(TotalQueueSize)><>(Value)
+    name: kafka_controller_$1_$2_$3
+  - pattern: kafka.controller<type=(KafkaController), name=(.+)><>(Value)
+    name: kafka_controller_$1_$2_$3
+  - pattern: kafka.controller<type=(ControllerStats), name=(.+)><>(Count)
+    name: kafka_controller_$1_$2_$3
+  - pattern : kafka.network<type=(Processor), name=(IdlePercent), networkProcessor=(.+)><>(Value)
+    name: kafka_network_$1_$2_$4
+    labels:
+      network_processor: $3
+  - pattern : kafka.network<type=(RequestMetrics), name=(.+), request=(.+)><>(Count|Value)
+    name: kafka_network_$1_$2_$4
+    labels:
+      request: $3
+  - pattern : kafka.network<type=(SocketServer), name=(.+)><>(Count|Value)
+    name: kafka_network_$1_$2_$3
+  - pattern : kafka.network<type=(RequestChannel), name=(.+)><>(Count|Value)
+    name: kafka_network_$1_$2_$3
+  - pattern: kafka.server<type=(.+), name=(.+), topic=(.+)><>(Count|OneMinuteRate)
+    name: kafka_server_$1_$2_$4
+    labels:
+      topic: $3
+  - pattern: kafka.server<type=(ReplicaFetcherManager), name=(.+), clientId=(.+)><>(Value)
+    name: kafka_server_$1_$2_$4
+    labels:
+      client_id: "$3"
+  - pattern: kafka.server<type=(DelayedOperationPurgatory), name=(.+), delayedOperation=(.+)><>(Value)
+    name: kafka_server_$1_$2_$3_$4
+  - pattern: kafka.server<type=(.+), name=(.+)><>(Count|Value|OneMinuteRate)
+    name: kafka_server_$1_total_$2_$3
+  - pattern: kafka.server<type=(.+)><>(queue-size)
+    name: kafka_server_$1_$2
+  - pattern: java.lang<type=(.+), name=(.+)><(.+)>(\w+)
+    name: java_lang_$1_$4_$3_$2
+  - pattern: java.lang<type=(.+), name=(.+)><>(\w+)
+    name: java_lang_$1_$3_$2
+  - pattern : java.lang<type=(.*)>
+  - pattern: kafka.log<type=(.+), name=(.+), topic=(.+), partition=(.+)><>Value
+    name: kafka_log_$1_$2
+    labels:
+      topic: $3
+      partition: $4

docker/prometheus.yml

@@ -31,6 +31,15 @@ scrape_configs:
       - source_labels: [__meta_docker_port_private]
         regex: 7000
         action: keep
+  - job_name: kafka
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 30s
+        filters:
+          - name: label
+            values:
+              - com.docker.compose.project=akvorado
+              - com.docker.compose.service=kafka-jmx-exporter
   - job_name: akvorado
     metrics_path: /api/v0/metrics
     docker_sd_configs: