kafka-monitor

kafka-monitor copied to clipboard

Hi Team, I am facing an issue with setting up the xinfra-monitor.properties I am receiving errors when I add Jetty service

} (com.linkedin.xinfra.monitor.services.OffsetCommitService)
Exception in thread "main" java.lang.ClassNotFoundException: com.linkedin.kmf.services.JettyService
	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:264)
	at com.linkedin.xinfra.monitor.XinfraMonitor.<init>(XinfraMonitor.java:74)
	at com.linkedin.xinfra.monitor.XinfraMonitor.main(XinfraMonitor.java:186)

# Copyright 2016 LinkedIn Corp. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
# file except in compliance with the License. You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# This properties file specifies the tests/services that XinfraMonitor
# should instantiate and run, together with the key/value pairs used to
# configure these tests/services. It should have the following format:
#
# {
#   "name1" : {
#     "type": TestClassName
#     "key1": value1,
#     "key2": value2,
#     ...
#   },
#   "name2" : {
#     "type": ServiceClassName
#     "key1": value1,
#     "key2": value2,
#     ...
#   },
#   ...
# }
#
# TestClassName can be canonical name or simple name of any class that implements
# interface com.linkedin.kmf.services.Test. These classes should be under
# package com.linkedin.kmf.tests.
#
# ServiceClassName can be canonical name or simple name of any class that implements
# interface com.linkedin.kmf.services.Service. These classes should be under
# package com.linkedin.kmf.services.
#
# Each test/service should be configured with class.name which can be either TestClassName
# or ServiceClassName. The key for the test/service in the json map is used as name to
# identify the test/service in the log or JMX metrics, which is useful if multiple
# test/service with the same class.name are run in the same Kafka Monitor process.
#
# If using Secure Socket Layer for security protocol, SSL properties must be defined under
# produce.producer.props, consume.consumer.props, as well as single-cluster-monitor props

{
  "single-cluster-monitor": {
    "class.name": "com.linkedin.xinfra.monitor.apps.SingleClusterMonitor",
    "topic": "xinfra-monitor-topic",
    "zookeeper.connect": "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181",
    "bootstrap.servers": "kafka1:9092,kafka2:9092,kafka3:9092",
    "request.timeout.ms": 9000,
    "produce.record.delay.ms": 100,
    "topic-management.topicManagementEnabled": true,
    "topic-management.topicCreationEnabled": true,
    "topic-management.replicationFactor" : 3,
    "topic-management.partitionsToBrokersRatio" : 2.0,
    "topic-management.rebalance.interval.ms" : 600000,
    "topic-management.preferred.leader.election.check.interval.ms" : 300000,
    "topic-management.topicFactory.props": {
    },
    "topic-management.topic.props": {
      "retention.ms": "3600000"
    },
    "produce.producer.props": {
      "client.id": "kmf-client-id"
    },

    "consume.latency.sla.ms": "20000",
    "consume.consumer.props": {
    }
  },

  "offset-commit-service": {
       "class.name": "com.linkedin.xinfra.monitor.services.OffsetCommitService",
       "zookeeper.connect": "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181",
       "bootstrap.servers": "kafka1:9092,kafka2:9092,kafka3:9092",
       "consumer.props": {
           "group.id": "target-consumer-group"
       }
  },

  "jolokia-service": {
    "class.name": "com.linkedin.xinfra.monitor.services.JolokiaService"
  },

 "jetty-service": {
    "class.name": "com.linkedin.kmf.services.JettyService",
    "jetty.port": 8080
  },

  "reporter-service": {
    "class.name": "com.linkedin.xinfra.monitor.services.DefaultMetricsReporterService",
    "report.interval.sec": 1,
    "report.metrics.list": [
        "kmf:type=kafka-monitor:offline-runnable-count",
        "kmf.services:type=produce-service,name=*:produce-availability-avg",
        "kmf.services:type=consume-service,name=*:consume-availability-avg",
        "kmf.services:type=produce-service,name=*:records-produced-total",
        "kmf.services:type=consume-service,name=*:records-consumed-total",
        "kmf.services:type=produce-service,name=*:records-produced-rate",
        "kmf.services:type=produce-service,name=*:produce-error-rate",
        "kmf.services:type=consume-service,name=*:consume-error-rate",
        "kmf.services:type=consume-service,name=*:records-lost-total",
        "kmf.services:type=consume-service,name=*:records-lost-rate",
        "kmf.services:type=consume-service,name=*:records-duplicated-total",
        "kmf.services:type=consume-service,name=*:records-delay-ms-avg",
        "kmf.services:type=commit-availability-service,name=*:offsets-committed-avg",
        "kmf.services:type=commit-availability-service,name=*:offsets-committed-total",
        "kmf.services:type=commit-availability-service,name=*:failed-commit-offsets-avg",
        "kmf.services:type=commit-availability-service,name=*:failed-commit-offsets-total",
        "kmf.services:type=commit-latency-service,name=*:commit-offset-latency-ms-avg",
        "kmf.services:type=commit-latency-service,name=*:commit-offset-latency-ms-max",
        "kmf.services:type=commit-latency-service,name=*:commit-offset-latency-ms-99th",
        "kmf.services:type=commit-latency-service,name=*:commit-offset-latency-ms-999th",
        "kmf.services:type=commit-latency-service,name=*:commit-offset-latency-ms-9999th",
        "kmf.services:type=cluster-topic-manipulation-service,name=*:topic-creation-metadata-propagation-ms-avg",
        "kmf.services:type=cluster-topic-manipulation-service,name=*:topic-creation-metadata-propagation-ms-max",
        "kmf.services:type=cluster-topic-manipulation-service,name=*:topic-deletion-metadata-propagation-ms-avg",
        "kmf.services:type=cluster-topic-manipulation-service,name=*:topic-deletion-metadata-propagation-ms-max",
        "kmf.services:type=offset-commit-service,name=*:offset-commit-availability-avg",
        "kmf.services:type=offset-commit-service,name=*:offset-commit-service-success-rate",
        "kmf.services:type=offset-commit-service,name=*:offset-commit-service-success-total",
        "kmf.services:type=offset-commit-service,name=*:offset-commit-service-failure-rate",
        "kmf.services:type=offset-commit-service,name=*:offset-commit-service-failure-total"
    ]
  },

  "cluster-topic-manipulation-service":{
     "class.name":"com.linkedin.xinfra.monitor.services.ClusterTopicManipulationService",
     "zookeeper.connect": "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181",
     "bootstrap.servers":"kafka1:9092,kafka2:9092,kafka3:9092",
     "topic": "xinfra-monitor-topic"
  },

#  Example produce-service to produce messages to cluster
#    "produce-service": {
#        "class.name": "com.linkedin.kmf.services.ProduceService",
#        "topic": "xinfra-monitor-topic",
#        "zookeeper.connect": "localhost:2181",
#        "bootstrap.servers": "localhost:9092",
#        "consume.latency.sla.ms": "20000",
#        "consume.consumer.props": {
#        }
#    },

#  Example consume-service to consume messages
#    "consume-service": {
#        "class.name": "com.linkedin.kmf.services.ConsumeService",
#        "topic": "xinfra-monitor-topic",
#        "zookeeper.connect": "localhost:2181",
#        "bootstrap.servers": "localhost:9092",
#        "consume.latency.sla.ms": "20000",
#        "consume.consumer.props": {
#        }
#    },

#  Example statsd-service to report metrics
#  "statsd-service": {
#      "class.name": "com.linkedin.xinfra.monitor.services.StatsdMetricsReporterService",
#      "report.statsd.host": "localhost",
#      "report.statsd.port": "8125",
#      "report.statsd.prefix": "xinfra-monitor",
#      "report.interval.sec": 1,
#      "report.metrics.list": [
#      "kmf.services:type=produce-service,name=*:produce-availability-avg",
#      "kmf.services:type=consume-service,name=*:consume-availability-avg"
#     ]
#  },

#  Example kafka-service to report metrics
  "reporter-kafka-service": {
    "class.name": "com.linkedin.xinfra.monitor.services.KafkaMetricsReporterService",
    "report.interval.sec": 3,
    "zookeeper.connect": "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181",
    "bootstrap.servers": "kafka1:9092,kafka2:9092,kafka3:9092",
    "topic": "xinfra-monitor-topic-metrics",
    "report.kafka.topic.replication.factor": 3,
    "report.metrics.list": [
      "kmf.services:type=produce-service,name=*:produce-availability-avg",
      "kmf.services:type=consume-service,name=*:consume-availability-avg",
      "kmf.services:type=produce-service,name=*:records-produced-total",
      "kmf.services:type=consume-service,name=*:records-consumed-total",
      "kmf.services:type=consume-service,name=*:records-lost-total",
      "kmf.services:type=consume-service,name=*:records-duplicated-total",
      "kmf.services:type=consume-service,name=*:records-delay-ms-avg",
      "kmf.services:type=produce-service,name=*:records-produced-rate",
      "kmf.services:type=produce-service,name=*:produce-error-rate",
      "kmf.services:type=consume-service,name=*:consume-error-rate"
    ]
  }


#  Example signalfx-service to report metrics
# "signalfx-service": {
#   "class.name": "com.linkedin.kmf.services.SignalFxMetricsReporterService",
#   "report.interval.sec": 1,
#   "report.metric.dimensions": {
#   },
#   "report.signalfx.url": "",
#   "report.signalfx.token" : ""
# }

}

If anyone can help me with using jetty service as documentation is not updated. Also, if the service is removed how I can access the UI Dashboard.

Thank you

This is your first issue in the repository. Thank you for raising this issue.' first issue

I am having the same issue

Having the same issue, even when I adjust the xinfra-monitor.properties file by taking into account name change/folder structure changes by adding:

"jetty-service": { "class.name": "com.linkedin.xinfra.monitor.services.JettyService", "jetty.port": 8000 },