diff --git a/docker/docker-compose-monitoring.yml b/docker/docker-compose-monitoring.yml index 103083bc2..8e40ca5ce 100644 --- a/docker/docker-compose-monitoring.yml +++ b/docker/docker-compose-monitoring.yml @@ -25,6 +25,17 @@ services: - traefik.http.routers.prometheus.rule=PathPrefix(`/prometheus`) - traefik.http.services.prometheus.loadbalancer.server.port=9090 + kafka-jmx-exporter: + image: bitnami/jmx-exporter:0 + restart: unless-stopped + depends_on: + - kafka + volumes: + - ./kafka-jmx-exporter-config.yaml:/etc/jmx-kafka/config.yaml:ro + command: + - "5556" + - /etc/jmx-kafka/config.yaml + kafka-ui: image: provectuslabs/kafka-ui:v0.5.0 restart: unless-stopped diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4a70543c2..d8a43bd01 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,6 +40,7 @@ services: - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT + - JMX_PORT=5555 restart: unless-stopped depends_on: - zookeeper diff --git a/docker/kafka-jmx-exporter-config.yaml b/docker/kafka-jmx-exporter-config.yaml new file mode 100644 index 000000000..322e53089 --- /dev/null +++ b/docker/kafka-jmx-exporter-config.yaml @@ -0,0 +1,61 @@ +--- +jmxUrl: service:jmx:rmi:///jndi/rmi://kafka:5555/jmxrmi +lowercaseOutputName: true +lowercaseOutputLabelNames: true +ssl: false + +# This is mostly stolen from Bitnami Helm chart for Kafka. +# https://github.com/bitnami/charts/blob/main/bitnami/kafka/templates/metrics/jmx-configmap.yaml +whitelistObjectNames: + - kafka.controller:* + - kafka.server:* + - java.lang:* + - kafka.network:* + - kafka.log:* +rules: + - pattern: kafka.controller<>(Value) + name: kafka_controller_$1_$2_$4 + labels: + broker_id: "$3" + - pattern: kafka.controller<>(Value) + name: kafka_controller_$1_$2_$3 + - pattern: kafka.controller<>(Value) + name: kafka_controller_$1_$2_$3 + - pattern: kafka.controller<>(Count) + name: kafka_controller_$1_$2_$3 + - pattern : kafka.network<>(Value) + name: kafka_network_$1_$2_$4 + labels: + network_processor: $3 + - pattern : kafka.network<>(Count|Value) + name: kafka_network_$1_$2_$4 + labels: + request: $3 + - pattern : kafka.network<>(Count|Value) + name: kafka_network_$1_$2_$3 + - pattern : kafka.network<>(Count|Value) + name: kafka_network_$1_$2_$3 + - pattern: kafka.server<>(Count|OneMinuteRate) + name: kafka_server_$1_$2_$4 + labels: + topic: $3 + - pattern: kafka.server<>(Value) + name: kafka_server_$1_$2_$4 + labels: + client_id: "$3" + - pattern: kafka.server<>(Value) + name: kafka_server_$1_$2_$3_$4 + - pattern: kafka.server<>(Count|Value|OneMinuteRate) + name: kafka_server_$1_total_$2_$3 + - pattern: kafka.server<>(queue-size) + name: kafka_server_$1_$2 + - pattern: java.lang<(.+)>(\w+) + name: java_lang_$1_$4_$3_$2 + - pattern: java.lang<>(\w+) + name: java_lang_$1_$3_$2 + - pattern : java.lang + - pattern: kafka.log<>Value + name: kafka_log_$1_$2 + labels: + topic: $3 + partition: $4 diff --git a/docker/prometheus.yml b/docker/prometheus.yml index 5e7fbe544..86950bc87 100644 --- a/docker/prometheus.yml +++ b/docker/prometheus.yml @@ -31,6 +31,15 @@ scrape_configs: - source_labels: [__meta_docker_port_private] regex: 7000 action: keep + - job_name: kafka + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 30s + filters: + - name: label + values: + - com.docker.compose.project=akvorado + - com.docker.compose.service=kafka-jmx-exporter - job_name: akvorado metrics_path: /api/v0/metrics docker_sd_configs: