title: Kafka Metrics exporter for producer and consumer
body: <p>I installed a Cloudera CDP cluster with 3 brokers. I export the metrics with the Prometheus JMX exporter. However, I cannot use the producer and consumer metrics in Prometheus. Do these client-side metrics have to be exported by the producer and consumer?</p>
  <p>The metrics for the Kafka broker are available. Is there anything missing or can one point me in the right direction?</p>
  <p>My Kafka exporter YAML looks like this:</p>
  <pre><code>#https://github.com/prometheus/jmx_exporter/blob/master/example_configs/kafka-2_0_0.yml
  lowercaseOutputName: true
  
  rules:
  # Special cases and very specific rules
  - pattern : kafka.server&lt;type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)&gt;&lt;&gt;Value
    name: kafka_server_$1_$2
    type: GAUGE
    labels:
      clientId: &quot;$3&quot;
      topic: &quot;$4&quot;
      partition: &quot;$5&quot;
  - pattern : kafka.server&lt;type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)&gt;&lt;&gt;Value
    name: kafka_server_$1_$2
    type: GAUGE
    labels:
      clientId: &quot;$3&quot;
      broker: &quot;$4:$5&quot;
  - pattern : kafka.coordinator.(\w+)&lt;type=(.+), name=(.+)&gt;&lt;&gt;Value
    name: kafka_coordinator_$1_$2_$3
    type: GAUGE
  
  
  - pattern: kafka.server&lt;type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)&gt;&lt;&gt;connections
    name: kafka_server_$1_connections_tls_info
    type: GAUGE
    labels:
      listener: &quot;$2&quot;
      networkProcessor: &quot;$3&quot;
      protocol: &quot;$4&quot;
      cipher: &quot;$5&quot;
  - pattern: kafka.server&lt;type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)&gt;&lt;&gt;connections
    name: kafka_server_$1_connections_software
    type: GAUGE
    labels:
      clientSoftwareName: &quot;$2&quot;
      clientSoftwareVersion: &quot;$3&quot;
      listener: &quot;$4&quot;
      networkProcessor: &quot;$5&quot;
  - pattern: &quot;kafka.server&lt;type=(.+), listener=(.+), networkProcessor=(.+)&gt;&lt;&gt;(.+):&quot;
    name: kafka_server_$1_$4
    type: GAUGE
    labels:
      listener: &quot;$2&quot;
      networkProcessor: &quot;$3&quot;
  - pattern: kafka.server&lt;type=(.+), listener=(.+), networkProcessor=(.+)&gt;&lt;&gt;(.+)
    name: kafka_server_$1_$4
    type: GAUGE
    labels:
      listener: &quot;$2&quot;
      networkProcessor: &quot;$3&quot;
  
  # Generic per-second counters with 0-2 key/value pairs
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)&gt;&lt;&gt;Count
    name: kafka_$1_$2_$3_total
    type: COUNTER
    labels:
      &quot;$4&quot;: &quot;$5&quot;
      &quot;$6&quot;: &quot;$7&quot;
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+)PerSec\w*, (.+)=(.+)&gt;&lt;&gt;Count
    name: kafka_$1_$2_$3_total
    type: COUNTER
    labels:
      &quot;$4&quot;: &quot;$5&quot;
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+)PerSec\w*&gt;&lt;&gt;Count
    name: kafka_$1_$2_$3_total
    type: COUNTER
  
  - pattern: kafka.server&lt;type=(.+), client-id=(.+)&gt;&lt;&gt;([a-z-]+)
    name: kafka_server_quota_$3
    type: GAUGE
    labels:
      resource: &quot;$1&quot;
      clientId: &quot;$2&quot;
  
  - pattern: kafka.server&lt;type=(.+), user=(.+), client-id=(.+)&gt;&lt;&gt;([a-z-]+)
    name: kafka_server_quota_$4
    type: GAUGE
    labels:
      resource: &quot;$1&quot;
      user: &quot;$2&quot;
      clientId: &quot;$3&quot;
  
  # Generic gauges with 0-2 key/value pairs
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)&gt;&lt;&gt;Value
    name: kafka_$1_$2_$3
    type: GAUGE
    labels:
      &quot;$4&quot;: &quot;$5&quot;
      &quot;$6&quot;: &quot;$7&quot;
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+), (.+)=(.+)&gt;&lt;&gt;Value
    name: kafka_$1_$2_$3
    type: GAUGE
    labels:
      &quot;$4&quot;: &quot;$5&quot;
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+)&gt;&lt;&gt;Value
    name: kafka_$1_$2_$3
    type: GAUGE
  
  # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
  #
  # Note that these are missing the '_sum' metric!
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)&gt;&lt;&gt;Count
    name: kafka_$1_$2_$3_count
    type: COUNTER
    labels:
      &quot;$4&quot;: &quot;$5&quot;
      &quot;$6&quot;: &quot;$7&quot;
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)&gt;&lt;&gt;(\d+)thPercentile
    name: kafka_$1_$2_$3
    type: GAUGE
    labels:
      &quot;$4&quot;: &quot;$5&quot;
      &quot;$6&quot;: &quot;$7&quot;
      quantile: &quot;0.$8&quot;
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+), (.+)=(.+)&gt;&lt;&gt;Count
    name: kafka_$1_$2_$3_count
    type: COUNTER
    labels:
      &quot;$4&quot;: &quot;$5&quot;
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+), (.+)=(.*)&gt;&lt;&gt;(\d+)thPercentile
    name: kafka_$1_$2_$3
    type: GAUGE
    labels:
      &quot;$4&quot;: &quot;$5&quot;
      quantile: &quot;0.$6&quot;
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+)&gt;&lt;&gt;Count
    name: kafka_$1_$2_$3_count
    type: COUNTER
  - pattern: kafka.(\w+)&lt;type=(.+), name=(.+)&gt;&lt;&gt;(\d+)thPercentile
    name: kafka_$1_$2_$3
    type: GAUGE
    labels:
      quantile: &quot;0.$4&quot;
    
  #Client
  - pattern : kafka.producer&lt;type=producer-metrics, client-id=(.+)&gt;&lt;&gt;(.+):\w*
    name: kafka_producer_$2
  
  - pattern : kafka.consumer&lt;type=consumer-metrics, client-id=(.+)&gt;&lt;&gt;(.+):\w*
    name: kafka_consumer_$2
  
  - pattern : kafka.consumer&lt;type=consumer-fetch-manager-metrics, client-id=(.+)&gt;&lt;&gt;(.+):\w*
    name: kafka_consumer_$2
    # &quot;kafka.consumer:type=app-info,client-id=*&quot;
    # &quot;kafka.producer:type=app-info,client-id=*&quot;
  - pattern: &quot;kafka.(.+)&lt;type=app-info, client-id=(.+)&gt;&lt;&gt;(.+): (.+)&quot;
    value: 1
    name: kafka_$1_app_info
    labels:
      client_type: $1
      client_id: $2
      $3: $4
    type: UNTYPED
  
  - pattern: &quot;kafka.(.+)&lt;type=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)&gt;&lt;&gt;(.+):&quot;
    name: kafka_$1_$2_$9
    type: GAUGE
    labels:
      client_type: $1
      $3: &quot;$4&quot;
      $5: &quot;$6&quot;
      $7: &quot;$8&quot;
  - pattern: &quot;kafka.(.+)&lt;type=(.+), (.+)=(.+), (.+)=(.+)&gt;&lt;&gt;(.+):&quot;
    name: kafka_$1_$2_$7
    type: GAUGE
    labels:
      client_type: $1
      $3: &quot;$4&quot;
      $5: &quot;$6&quot;
    # &quot;kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*&quot;
    # &quot;kafka.consumer:type=consumer-metrics,client-id=*&quot;
    # &quot;kafka.producer:type=producer-metrics,client-id=*&quot;
  - pattern: &quot;kafka.(.+)&lt;type=(.+), (.+)=(.+)&gt;&lt;&gt;(.+):&quot;
    name: kafka_$1_$2_$5
    type: GAUGE
    labels:
      client_type: $1
      $3: &quot;$4&quot;
  - pattern: &quot;kafka.(.+)&lt;type=(.+)&gt;&lt;&gt;(.+):&quot;
    name: kafka_$1_$2_$3
    labels:
      client_type: $1
  </code></pre>
  <p>Thank you for the help!</p>