资源规划

组件 bigdata-node1 bigdata-node2 bigdata-node3
OS centos7.6 centos7.6 centos7.6
JDK jvm jvm jvm
Zookeeper QuorumPeerMain QuorumPeerMain QuorumPeerMain
Influxdb N.A Influxdb N.A
JmxTrans N.A jmxtrans N.A
Grafana N.A Grafana N.A

JMX监控

开启JMX监控（zk默认会在ipv4和ipv6上各开一个端口用来支持jmx调用）：

  1. vi ~/modules/zookeeper-3.4.10/bin/zkServer.sh

配置如下：

  1. # JMX Config
  2. JMXLOCALONLY=false
  3. JMXDISABLE=false
  4. JMXPORT=8999
  5. JMXAUTH=false
  6. JMXSSL=false

验证：

  1. sudo netstat -lnpt |grep 8999

也可以通过jconsole工具远程连接JMX。
image.pngimage.png
image.pngimage.png
zk监控文件配置（根据JmxTrans部署方式调整zk-jmxtrans.json，如果**JmxTrans部署在一个节点上，则可以编写多个json文件，注意修改IP、alias信息；如果JmxTrans是分布式部署在zk的节点上，则IP可以统一为localhost，这样配置文件即可保持一致**）：

  1. {
  2.   "servers": [
  3.     {
  4.       "port": 8999,
  5.       "host": "localhost",
  6.       "alias": "zk",
  7.       "queries": [
  8.         {
  9.           "resultAlias": "zkQuorum",
  10.           "outputWriters" : [
  11.             {
  12.               "@class" : "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
  13.               "url" : "http://192.168.0.102:8086/",
  14.               "database" : "zkdb",
  15.               "username": "root",
  16.               "password": "123456"
  17.             } ],
  18.           "obj": "org.apache.ZooKeeperService:name0=ReplicatedServer_id*",
  19.           "attr": [
  20.             "QuorumSize"
  21.           ]
  22.         },
  23.         {
  24.           "resultAlias": "zkCluster",
  25.           "outputWriters" : [
  26.             {
  27.               "@class" : "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
  28.               "url" : "http://192.168.0.102:8086/",
  29.               "database" : "zkdb",
  30.               "username": "root",
  31.               "password": "123456"
  32.             } ],
  33.           "obj": "org.apache.ZooKeeperService:name0=ReplicatedServer_id*,name1=replica*,name2=*",
  34.           "attr": [
  35.             "MaxClientCnxnsPerHost",
  36.             "MaxRequestLatency",
  37.             "AvgRequestLatency",
  38.             "MinRequestLatency",
  39.             "MaxSessionTimeout",
  40.             "MinSessionTimeout",
  41.             "OutstandingRequests",
  42.             "PacketsReceived",
  43.             "PacketsSent",
  44.             "PendingRevalidationCount",
  45.             "TickTime"
  46.           ]
  47.         },
  48.         {
  49.           "resultAlias": "zkCount",
  50.           "outputWriters" : [
  51.             {
  52.               "@class" : "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
  53.               "url" : "http://192.168.0.102:8086/",
  54.               "database" : "zkdb",
  55.               "username": "root",
  56.               "password": "123456"
  57.             } ],
  58.           "obj": "org.apache.ZooKeeperService:name0=ReplicatedServer_id*,name1=replica*,name2=*,name3=InMemoryDataTree",
  59.           "attr": [
  60.             "NodeCount",
  61.             "WatchCount"
  62.           ]
  63.         },
  64.         {
  65.           "resultAlias": "Clients",
  66.           "outputWriters" : [
  67.             {
  68.               "@class" : "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
  69.               "url" : "http://192.168.0.102:8086/",
  70.               "database" : "zkdb",
  71.               "username": "root",
  72.               "password": "123456"
  73.             } ],
  74.           "obj": "org.apache.ZooKeeperService:name0=ReplicatedServer_id*,name1=replica*,name2=*,name3=Connections,name4=*,name5=*",
  75.           "attr": [
  76.             "AvgLatency",
  77.             "LastLatency",
  78.             "MaxLatency",
  79.             "MinLatency",
  80.             "OutstandingRequests",
  81.             "PacketsReceived",
  82.             "PacketsSent"
  83.           ]
  84.         },
  85.         {
  86.           "resultAlias": "Memory",
  87.           "outputWriters" : [
  88.             {
  89.               "@class" : "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
  90.               "url" : "http://192.168.0.102:8086/",
  91.               "database" : "zkdb",
  92.               "username": "root",
  93.               "password": "123456"
  94.             } ],
  95.           "obj": "java.lang:type=Memory",
  96.           "attr": [
  97.             "HeapMemoryUsage",
  98.             "NonHeapMemoryUsage"
  99.           ]
  100.         },
  101.         {
  102.           "resultAlias": "Process",
  103.           "outputWriters" : [
  104.             {
  105.               "@class" : "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
  106.               "url" : "http://192.168.0.102:8086/",
  107.               "database" : "zkdb",
  108.               "username": "root",
  109.               "password": "123456"
  110.             } ],
  111.           "obj": "java.lang:type=OperatingSystem",
  112.           "attr": [
  113.             "OpenFileDescriptorCount",
  114.             "ProcessCpuLoad"
  115.           ]
  116.         },
  117.         {
  118.           "resultAlias": "Threading",
  119.           "outputWriters" : [
  120.             {
  121.               "@class" : "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
  122.               "url" : "http://192.168.0.102:8086/",
  123.               "database" : "zkdb",
  124.               "username": "root",
  125.               "password": "123456"
  126.             } ],
  127.           "obj": "java.lang:type=Threading",
  128.           "attr": [
  129.             "ThreadCount"
  130.           ]
  131.         }
  132.       ],
  133.       "numQueryThreads": 2
  134.     }
  135.   ]
  136. }

注意：建议JmxTrans启动前，在Influxdb中配置好，zk存储数据库，已经数据库用户口令，以便于采集和Grafana中使用。

  1. -- 启动Influxdb
  2. influx
  3. -- 创建用户，并分配权限
  4. create user "root" with password '123456' with all privileges;
  5. create database zkdb;

参考

【jmxtrans+influxdb+grafana监控zookeeper实战】https://www.jianshu.com/p/a911ec9effb4
【ZooKeeper官网JXM指标说明】http://zookeeper.apache.org/doc/r3.4.6/zookeeperJMX.html