Categories
DevOps

How to enable ZooKeeper the four-letter words

Enable the four-letter words.

Keep in mind that this functionality will be deprecated, please use AdminServer instead.

Enable the four letter words inside zoo.cfg configuration file.

$ sudo -u zookeeper cat /opt/zookeeper/zookeeper/conf/zoo.cfg 
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/opt/zookeeper/zookeeper/data
clientPort=2181
server.1=zookeeper1.example.org:2888:3888
server.2=zookeeper2.example.org:2888:3888
server.3=zookeeper3.example.org:2888:3888

4lw.commands.whitelist=*

You don’t need to enable every available command, use comma to list only needed ones.

Alternatively use JVMFLAGS.

$ sudo -u zookeeper cat /opt/zookeeper/zookeeper/conf/zookeeper-env.sh
JVMFLAGS="-Dzookeeper.4lw.commands.whitelist=*"

Restart ZooKeeper service.

$ systemctl restart zookeeper.service

Display configuration.

$ echo conf | nc zookeeper3.example.org 2181
clientPort=2181
secureClientPort=-1
dataDir=/opt/zookeeper/zookeeper/data/version-2
dataDirSize=67110138
dataLogDir=/opt/zookeeper/zookeeper/data/version-2
dataLogSize=67110138
tickTime=2000
maxClientCnxns=60
minSessionTimeout=4000
maxSessionTimeout=40000
clientPortListenBacklog=-1
serverId=3
initLimit=10
syncLimit=5
electionAlg=3
electionPort=3888
quorumPort=2888
peerType=0
membership: 
server.1=zookeeper1.example.org:2888:3888:participant
server.2=zookeeper2.example.org:2888:3888:participant
server.3=zookeeper3.example.org:2888:3888:participant

List connection details to this server.

$ echo cons | nc zookeeper3.example.org 2181
 /172.16.0.113:53288[0](queued=0,recved=1,sent=0)
 /172.16.0.111:33890[1](queued=0,recved=2,sent=2,sid=0x30000b41d990000,lop=EXIS,est=1623450894071,to=30000,lcxid=0x1,lzxid=0x10000007c,lresp=11994794,llat=2,minlat=0,avglat=6,maxlat=11)

Reset connection statistics.

$ echo crst | nc zookeeper3.example.org 2181
Connection stats reset.

Display outstanding sessions and ephemeral nodes.

$ echo dump | nc zookeeper3.example.org 2181
SessionTracker dump:
Global Sessions(1):
0x30000b41d990000       30000ms
ephemeral nodes dump:
Sessions with Ephemerals (0):
Connections dump:
Connections Sets (4)/(2):
0 expire at Fri Jun 11 22:35:26 UTC 2021:
1 expire at Fri Jun 11 22:35:36 UTC 2021:
        ip: /172.16.0.113:53290 sessionId: 0x0
0 expire at Fri Jun 11 22:35:46 UTC 2021:
1 expire at Fri Jun 11 22:35:56 UTC 2021:
        ip: /172.16.0.111:33890 sessionId: 0x30000b41d990000

Display environment settings.

$ echo envi | nc zookeeper3.example.org 2181
Environment:
zookeeper.version=3.7.0-e3704b390a6697bfdf4b0bef79e3da7a4f6bac4b, built on 2021-03-17 09:46 UTC
host.name=zookeeper3
java.version=11.0.11
java.vendor=Debian
java.home=/usr/lib/jvm/java-11-openjdk-amd64
java.class.path=/opt/zookeeper/zookeeper/bin/../zookeeper-server/target/classes:/opt/zookeeper/zookeeper/bin/../build/classes:/opt/zookeeper/zookeeper/bin/../zookeeper-server/target/lib/*.jar:/opt/zookeeper/zookeeper/bin/../build/lib/*.jar:/opt/zookeeper/zookeeper/bin/../lib/zookeeper-prometheus-metrics-3.7.0.jar:/opt/zookeeper/zookeeper/bin/../lib/zookeeper-jute-3.7.0.jar:/opt/zookeeper/zookeeper/bin/../lib/zookeeper-3.7.0.jar:/opt/zookeeper/zookeeper/bin/../lib/snappy-java-1.1.7.7.jar:/opt/zookeeper/zookeeper/bin/../lib/slf4j-log4j12-1.7.30.jar:/opt/zookeeper/zookeeper/bin/../lib/slf4j-api-1.7.30.jar:/opt/zookeeper/zookeeper/bin/../lib/simpleclient_servlet-0.9.0.jar:/opt/zookeeper/zookeeper/bin/../lib/simpleclient_hotspot-0.9.0.jar:/opt/zookeeper/zookeeper/bin/../lib/simpleclient_common-0.9.0.jar:/opt/zookeeper/zookeeper/bin/../lib/simpleclient-0.9.0.jar:/opt/zookeeper/zookeeper/bin/../lib/netty-transport-native-unix-common-4.1.59.Final.jar:/opt/zookeeper/zookeeper/bin/../lib/netty-transport-native-epoll-4.1.59.Final.jar:/opt/zookeeper/zookeeper/bin/../lib/netty-transport-4.1.59.Final.jar:/opt/zookeeper/zookeeper/bin/../lib/netty-resolver-4.1.59.Final.jar:/opt/zookeeper/zookeeper/bin/../lib/netty-handler-4.1.59.Final.jar:/opt/zookeeper/zookeeper/bin/../lib/netty-common-4.1.59.Final.jar:/opt/zookeeper/zookeeper/bin/../lib/netty-codec-4.1.59.Final.jar:/opt/zookeeper/zookeeper/bin/../lib/netty-buffer-4.1.59.Final.jar:/opt/zookeeper/zookeeper/bin/../lib/metrics-core-4.1.12.1.jar:/opt/zookeeper/zookeeper/bin/../lib/log4j-1.2.17.jar:/opt/zookeeper/zookeeper/bin/../lib/jline-2.14.6.jar:/opt/zookeeper/zookeeper/bin/../lib/jetty-util-ajax-9.4.38.v20210224.jar:/opt/zookeeper/zookeeper/bin/../lib/jetty-util-9.4.38.v20210224.jar:/opt/zookeeper/zookeeper/bin/../lib/jetty-servlet-9.4.38.v20210224.jar:/opt/zookeeper/zookeeper/bin/../lib/jetty-server-9.4.38.v20210224.jar:/opt/zookeeper/zookeeper/bin/../lib/jetty-security-9.4.38.v20210224.jar:/opt/zookeeper/zookeeper/bin/../lib/jetty-io-9.4.38.v20210224.jar:/opt/zookeeper/zookeeper/bin/../lib/jetty-http-9.4.38.v20210224.jar:/opt/zookeeper/zookeeper/bin/../lib/javax.servlet-api-3.1.0.jar:/opt/zookeeper/zookeeper/bin/../lib/jackson-databind-2.10.5.1.jar:/opt/zookeeper/zookeeper/bin/../lib/jackson-core-2.10.5.jar:/opt/zookeeper/zookeeper/bin/../lib/jackson-annotations-2.10.5.jar:/opt/zookeeper/zookeeper/bin/../lib/commons-cli-1.4.jar:/opt/zookeeper/zookeeper/bin/../lib/audience-annotations-0.12.0.jar:/opt/zookeeper/zookeeper/bin/../zookeeper-*.jar:/opt/zookeeper/zookeeper/bin/../zookeeper-server/src/main/resources/lib/*.jar:/opt/zookeeper/zookeeper/conf:
java.library.path=/usr/java/packages/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib
java.io.tmpdir=/tmp
java.compiler=
os.name=Linux
os.arch=amd64
os.version=5.10.0-3-amd64
user.name=zookeeper
user.home=/opt/zookeeper
user.dir=/opt/zookeeper
os.memory.free=6MB
os.memory.max=966MB
os.memory.total=14MB

My favorite four-letter word to test the server state.

$ echo ruok | nc zookeeper3.example.org 2181;echo
imok

Reset server statistics.

$ echo srst | nc zookeeper3.example.org 2181
Server stats reset.

Lists server statistics.

$ echo srvr | nc zookeeper3.example.org 2181
Zookeeper version: 3.7.0-e3704b390a6697bfdf4b0bef79e3da7a4f6bac4b, built on 2021-03-17 09:46 UTC
Latency min/avg/max: 0/1.5/11
Received: 21
Sent: 20
Connections: 2
Outstanding: 0
Zxid: 0x10000007d
Mode: follower
Node count: 6

Display server details and connected clients.

$ echo stat | nc zookeeper3.example.org 2181
Zookeeper version: 3.7.0-e3704b390a6697bfdf4b0bef79e3da7a4f6bac4b, built on 2021-03-17 09:46 UTC
Clients:
 /172.16.0.113:53300[0](queued=0,recved=1,sent=0)
 /172.16.0.111:33890[1](queued=0,recved=14,sent=14)

Latency min/avg/max: 0/1.5/11
Received: 24
Sent: 23
Connections: 2
Outstanding: 0
Zxid: 0x10000007d
Mode: follower
Node count: 6

Lists watches on the server.

$ echo wchs | nc zookeeper3.example.org 2181
1 connections watching 1 paths
Total watches:1

List detailed information on watches for the server, by session.

$ echo wchc | nc zookeeper3.example.org 2181
0x30000b41d990000
        /app

Display the total size of snapshot and log files in bytes.

$ echo dirs | nc zookeeper3.example.org 2181
datadir_size: 134219018
logdir_size: 134219018

List detailed information on watches for the server, by path.

$ echo wchp | nc zookeeper3.example.org 2181

/app
        0x30000b41d990000

Outputs a list of variables that could be used for monitoring the health of the cluster.

$ echo mntr | nc zookeeper3.example.org 2181
zk_version      3.7.0-e3704b390a6697bfdf4b0bef79e3da7a4f6bac4b, built on 2021-03-17 09:46 UTC
zk_server_state follower                         
zk_peer_state   following - broadcast             
zk_ephemerals_count     0                         
zk_num_alive_connections        2                 
zk_avg_latency  1.1364                   
zk_outstanding_requests 0                     
zk_znode_count  6                                
zk_global_sessions      1                  
zk_non_mtls_remote_conn_count   0         
zk_last_client_response_size    16         
zk_packets_sent 36                       
zk_packets_received     37                         
zk_max_client_response_size     84               
zk_connection_drop_probability  0.0               
zk_watch_count  1                                
zk_auth_failed_count    0                         
zk_min_latency  0                                
zk_max_file_descriptor_count    524288            
zk_approximate_data_size        219                
zk_open_file_descriptor_count   62                
zk_local_sessions       0                        
zk_uptime       394126                           
zk_max_latency  11                               
zk_outstanding_tls_handshake    0            
zk_min_client_response_size     16         
zk_non_mtls_local_conn_count    0          
zk_non_mtls_local_conn_count    0
zk_quorum_size  3                          
zk_proposal_count       0                  
zk_watch_bytes  0                           
zk_outstanding_changes_removed  0          
zk_throttled_ops        0                  
zk_stale_requests_dropped       0          
zk_large_requests_rejected      0         
zk_insecure_admin_count 0                 
zk_connection_rejected  0
zk_cnxn_closed_without_zk_server_running        0
zk_sessionless_connections_expired      0
zk_looking_count        1
zk_dead_watchers_queued 0
zk_stale_requests       0
zk_connection_drop_count        0
zk_learner_proposal_received_count      5
zk_digest_mismatches_count      0
zk_dead_watchers_cleared        0
zk_response_packet_cache_hits   0
zk_bytes_received_count 370
zk_add_dead_watcher_stall_time  0
zk_request_throttle_wait_count  0
zk_requests_not_forwarded_to_commit_processor   0
zk_response_packet_cache_misses 0
zk_ensemble_auth_success        0
zk_prep_processor_request_queued        0
zk_learner_commit_received_count        5
zk_stale_replies        0
zk_connection_request_count     1
zk_response_bytes       84
zk_ensemble_auth_fail   0
zk_diff_count   0
zk_response_packet_get_children_cache_misses    0
zk_connection_revalidate_count  0
zk_quit_leading_due_to_disloyal_voter   0
zk_snap_count   0
zk_unrecoverable_error_count    0
zk_unsuccessful_handshake       0
zk_commit_count 0
zk_stale_sessions_expired       0
zk_response_packet_get_children_cache_hits      0
zk_sync_processor_request_queued        5
zk_outstanding_changes_queued   0
zk_request_commit_queued        5
zk_ensemble_auth_skip   0
zk_skip_learner_request_to_next_processor_count 0
zk_tls_handshake_exceeded       0
zk_revalidate_count     0
zk_avg_socket_closing_time      0.0
zk_min_socket_closing_time      0
zk_max_socket_closing_time      0
zk_cnt_socket_closing_time      0
zk_sum_socket_closing_time      0
zk_avg_proposal_process_time    0.0
zk_min_proposal_process_time    0
zk_max_proposal_process_time    0
zk_cnt_proposal_process_time    0
zk_sum_proposal_process_time    0
zk_avg_leader_unavailable_time  0.0
zk_min_leader_unavailable_time  0
zk_max_leader_unavailable_time  0
zk_cnt_leader_unavailable_time  0
zk_sum_leader_unavailable_time  0
zk_avg_node_created_watch_count 0.0
zk_min_node_created_watch_count 0
zk_max_node_created_watch_count 0
zk_cnt_node_created_watch_count 0
zk_sum_node_created_watch_count 0
zk_avg_session_queues_drained   0.2
zk_min_session_queues_drained   0
zk_max_session_queues_drained   1
zk_cnt_session_queues_drained   5
zk_sum_session_queues_drained   1
zk_avg_write_commit_proc_req_queued     0.0741
zk_min_write_commit_proc_req_queued     0
zk_max_write_commit_proc_req_queued     1
zk_cnt_write_commit_proc_req_queued     27
zk_sum_write_commit_proc_req_queued     2
zk_avg_connection_token_deficit 0.0
zk_min_connection_token_deficit 0
zk_max_connection_token_deficit 0
zk_cnt_connection_token_deficit 1
zk_sum_connection_token_deficit 0
zk_avg_read_commit_proc_req_queued      0.7778
zk_min_read_commit_proc_req_queued      0
zk_max_read_commit_proc_req_queued      1
zk_cnt_read_commit_proc_req_queued      27
zk_sum_read_commit_proc_req_queued      21
zk_avg_node_deleted_watch_count 0.0
zk_min_node_deleted_watch_count 0
zk_max_node_deleted_watch_count 0
zk_cnt_node_deleted_watch_count 0
zk_sum_node_deleted_watch_count 0
zk_avg_startup_txns_load_time   10.0
zk_min_startup_txns_load_time   10
zk_max_startup_txns_load_time   10
zk_cnt_startup_txns_load_time   1
zk_sum_startup_txns_load_time   10
zk_avg_sync_processor_queue_size        0.0
zk_min_sync_processor_queue_size        0
zk_max_sync_processor_queue_size        0
zk_cnt_sync_processor_queue_size        6
zk_sum_sync_processor_queue_size        0
zk_avg_follower_sync_time       36.0
zk_min_follower_sync_time       36
zk_max_follower_sync_time       36
zk_cnt_follower_sync_time       1
zk_sum_follower_sync_time       36
zk_avg_prep_processor_queue_size        0.0
zk_min_prep_processor_queue_size        0
zk_max_prep_processor_queue_size        0
zk_cnt_prep_processor_queue_size        0
zk_sum_prep_processor_queue_size        0
zk_avg_fsynctime        2.0
zk_min_fsynctime        0
zk_max_fsynctime        10
zk_cnt_fsynctime        5
zk_sum_fsynctime        10
zk_avg_inflight_snap_count      0.0
zk_min_inflight_snap_count      0
zk_max_inflight_snap_count      0
zk_cnt_inflight_snap_count      0
zk_sum_inflight_snap_count      0
zk_avg_reads_issued_from_session_queue  0.0
zk_min_reads_issued_from_session_queue  0
zk_max_reads_issued_from_session_queue  0
zk_cnt_reads_issued_from_session_queue  5
zk_sum_reads_issued_from_session_queue  0
zk_avg_learner_request_processor_queue_size     0.0
zk_min_learner_request_processor_queue_size     0
zk_max_learner_request_processor_queue_size     0
zk_cnt_learner_request_processor_queue_size     23
zk_sum_learner_request_processor_queue_size     0
zk_avg_snapshottime     0.0
zk_min_snapshottime     0
zk_max_snapshottime     0
zk_cnt_snapshottime     0
zk_sum_snapshottime     0
zk_avg_unavailable_time 95.0
zk_min_unavailable_time 95
zk_max_unavailable_time 95
zk_cnt_unavailable_time 1
zk_sum_unavailable_time 95
zk_avg_startup_txns_loaded      120.0
zk_min_startup_txns_loaded      120
zk_max_startup_txns_loaded      120
zk_cnt_startup_txns_loaded      1
zk_sum_startup_txns_loaded      120
zk_avg_reads_after_write_in_session_queue       0.0
zk_min_reads_after_write_in_session_queue       0
zk_max_reads_after_write_in_session_queue       0
zk_cnt_reads_after_write_in_session_queue       1
zk_sum_reads_after_write_in_session_queue       0
zk_avg_requests_in_session_queue        1.0
zk_min_requests_in_session_queue        1
zk_max_requests_in_session_queue        1
zk_cnt_requests_in_session_queue        2
zk_sum_requests_in_session_queue        2
zk_avg_write_commit_proc_issued 1.0
zk_min_write_commit_proc_issued 1
zk_max_write_commit_proc_issued 1
zk_cnt_write_commit_proc_issued 5
zk_sum_write_commit_proc_issued 5
zk_avg_prep_process_time        0.0
zk_min_prep_process_time        0
zk_max_prep_process_time        0
zk_cnt_prep_process_time        0
zk_sum_prep_process_time        0
zk_avg_pending_session_queue_size       1.0
zk_min_pending_session_queue_size       1
zk_max_pending_session_queue_size       1
zk_cnt_pending_session_queue_size       1
zk_sum_pending_session_queue_size       1
zk_avg_time_waiting_empty_pool_in_commit_processor_read_ms      0.0
zk_min_time_waiting_empty_pool_in_commit_processor_read_ms      0
zk_max_time_waiting_empty_pool_in_commit_processor_read_ms      0
zk_cnt_time_waiting_empty_pool_in_commit_processor_read_ms      5
zk_sum_time_waiting_empty_pool_in_commit_processor_read_ms      0
zk_avg_commit_process_time      0.4074
zk_min_commit_process_time      0
zk_max_commit_process_time      5
zk_cnt_commit_process_time      27
zk_sum_commit_process_time      11
zk_avg_dbinittime       19.0
zk_min_dbinittime       19
zk_max_dbinittime       19
zk_cnt_dbinittime       1
zk_sum_dbinittime       19
zk_avg_inflight_diff_count      0.0
zk_min_inflight_diff_count      0
zk_max_inflight_diff_count      0
zk_cnt_inflight_diff_count      0
zk_sum_inflight_diff_count      0
zk_avg_netty_queued_buffer_capacity     0.0
zk_min_netty_queued_buffer_capacity     0
zk_max_netty_queued_buffer_capacity     0
zk_cnt_netty_queued_buffer_capacity     0
zk_sum_netty_queued_buffer_capacity     0
zk_avg_election_time    46.0
zk_min_election_time    46
zk_max_election_time    46
zk_cnt_election_time    1
zk_sum_election_time    46
zk_avg_commit_commit_proc_req_queued    0.1852
zk_min_commit_commit_proc_req_queued    0
zk_max_commit_commit_proc_req_queued    1
zk_cnt_commit_commit_proc_req_queued    27
zk_sum_commit_commit_proc_req_queued    5
zk_avg_sync_processor_batch_size        1.0
zk_min_sync_processor_batch_size        1
zk_max_sync_processor_batch_size        1
zk_cnt_sync_processor_batch_size        5
zk_sum_sync_processor_batch_size        5
zk_avg_node_children_watch_count        0.0
zk_min_node_children_watch_count        0
k_max_node_children_watch_count        0
zk_cnt_node_children_watch_count        0
zk_sum_node_children_watch_count        0
zk_avg_write_batch_time_in_commit_processor     1.0
zk_min_write_batch_time_in_commit_processor     0
zk_max_write_batch_time_in_commit_processor     5
zk_cnt_write_batch_time_in_commit_processor     5
zk_sum_write_batch_time_in_commit_processor     5
zk_avg_read_commit_proc_issued  0.7778
zk_min_read_commit_proc_issued  0
zk_max_read_commit_proc_issued  1
zk_cnt_read_commit_proc_issued  27
zk_sum_read_commit_proc_issued  21
zk_avg_concurrent_request_processing_in_commit_processor        0.0
zk_min_concurrent_request_processing_in_commit_processor        0
zk_max_concurrent_request_processing_in_commit_processor        0
zk_cnt_concurrent_request_processing_in_commit_processor        0
zk_sum_concurrent_request_processing_in_commit_processor        0
zk_avg_node_changed_watch_count 0.0
zk_min_node_changed_watch_count 0
zk_max_node_changed_watch_count 0
zk_cnt_node_changed_watch_count 0
zk_sum_node_changed_watch_count 0
zk_avg_sync_process_time        0.2
zk_min_sync_process_time        0
zk_max_sync_process_time        1
zk_cnt_sync_process_time        5
zk_sum_sync_process_time        1
zk_avg_startup_snap_load_time   6.0
zk_min_startup_snap_load_time   6
zk_max_startup_snap_load_time   6
zk_cnt_startup_snap_load_time   1
zk_sum_startup_snap_load_time   6
zk_avg_prep_processor_queue_time_ms     0.0
zk_min_prep_processor_queue_time_ms     0
zk_max_prep_processor_queue_time_ms     0
zk_cnt_prep_processor_queue_time_ms     0
zk_sum_prep_processor_queue_time_ms     0
zk_p50_prep_processor_queue_time_ms     0
zk_p95_prep_processor_queue_time_ms     0
zk_p99_prep_processor_queue_time_ms     0
zk_p999_prep_processor_queue_time_ms    0
zk_avg_jvm_pause_time_ms        0.0
zk_min_jvm_pause_time_ms        0
zk_max_jvm_pause_time_ms        0
zk_cnt_jvm_pause_time_ms        0
zk_sum_jvm_pause_time_ms        0
zk_p50_jvm_pause_time_ms        0
zk_p95_jvm_pause_time_ms        0
zk_p99_jvm_pause_time_ms        0
zk_p999_jvm_pause_time_ms       0
zk_avg_close_session_prep_time  0.0
zk_min_close_session_prep_time  0
zk_max_close_session_prep_time  0
zk_cnt_close_session_prep_time  0
zk_sum_close_session_prep_time  0
zk_p50_close_session_prep_time  0
zk_p95_close_session_prep_time  0
zk_p99_close_session_prep_time  0
zk_p999_close_session_prep_time 0
zk_avg_read_commitproc_time_ms  0.2857
zk_min_read_commitproc_time_ms  0
zk_max_read_commitproc_time_ms  1
zk_cnt_read_commitproc_time_ms  21
zk_sum_read_commitproc_time_ms  6
zk_p50_read_commitproc_time_ms  0
zk_p95_read_commitproc_time_ms  1
zk_p99_read_commitproc_time_ms  1
zk_p999_read_commitproc_time_ms 1
zk_avg_updatelatency    11.0
zk_min_updatelatency    11
zk_max_updatelatency    11
zk_cnt_updatelatency    1
zk_sum_updatelatency    11
zk_p50_updatelatency    11
zk_p95_updatelatency    11
zk_p99_updatelatency    11
zk_p999_updatelatency   11
zk_avg_local_write_committed_time_ms    1.0
zk_min_local_write_committed_time_ms    1
zk_max_local_write_committed_time_ms    1
zk_cnt_local_write_committed_time_ms    1
zk_sum_local_write_committed_time_ms    1
zk_p50_local_write_committed_time_ms    1
zk_p95_local_write_committed_time_ms    1
zk_p99_local_write_committed_time_ms    1
zk_p999_local_write_committed_time_ms   1
zk_avg_request_throttle_queue_time_ms   0.1818
zk_min_request_throttle_queue_time_ms   0
zk_max_request_throttle_queue_time_ms   1
zk_cnt_request_throttle_queue_time_ms   22
zk_sum_request_throttle_queue_time_ms   4
zk_p50_request_throttle_queue_time_ms   0
zk_p95_request_throttle_queue_time_ms   1
zk_p99_request_throttle_queue_time_ms   1
zk_p999_request_throttle_queue_time_ms  1
zk_avg_readlatency      0.6667
zk_min_readlatency      0
zk_max_readlatency      2
zk_cnt_readlatency      21
zk_sum_readlatency      14
zk_p50_readlatency      1
zk_p95_readlatency      2
zk_p99_readlatency      2
zk_p999_readlatency     2
zk_avg_quorum_ack_latency       0.0
zk_min_quorum_ack_latency       0
zk_max_quorum_ack_latency       0
zk_cnt_quorum_ack_latency       0
zk_sum_quorum_ack_latency       0
zk_p50_quorum_ack_latency       0
zk_p95_quorum_ack_latency       0
zk_p99_quorum_ack_latency       0
zk_p999_quorum_ack_latency      0
zk_avg_om_commit_process_time_ms        0.0
zk_min_om_commit_process_time_ms        0
zk_max_om_commit_process_time_ms        0
zk_cnt_om_commit_process_time_ms        0
zk_sum_om_commit_process_time_ms        0
zk_p50_om_commit_process_time_ms        0
zk_p95_om_commit_process_time_ms        0
zk_p99_om_commit_process_time_ms        0
zk_p999_om_commit_process_time_ms       0
zk_avg_read_final_proc_time_ms  0.2381
zk_min_read_final_proc_time_ms  0
zk_max_read_final_proc_time_ms  2
zk_cnt_read_final_proc_time_ms  21
zk_sum_read_final_proc_time_ms  5
zk_p50_read_final_proc_time_ms  0
zk_p95_read_final_proc_time_ms  2
zk_p99_read_final_proc_time_ms  2
zk_p999_read_final_proc_time_ms 2
zk_avg_commit_propagation_latency       5.4
zk_min_commit_propagation_latency       3
zk_max_commit_propagation_latency       11
zk_cnt_commit_propagation_latency       5
zk_sum_commit_propagation_latency       27
zk_p50_commit_propagation_latency       4
zk_p95_commit_propagation_latency       11
zk_p99_commit_propagation_latency       11
zk_p999_commit_propagation_latency      11
zk_avg_dead_watchers_cleaner_latency    0.0
zk_min_dead_watchers_cleaner_latency    0
zk_max_dead_watchers_cleaner_latency    0
zk_cnt_dead_watchers_cleaner_latency    0
zk_sum_dead_watchers_cleaner_latency    0
zk_p50_dead_watchers_cleaner_latency    0
zk_p95_dead_watchers_cleaner_latency    0
zk_p99_dead_watchers_cleaner_latency    0
zk_p999_dead_watchers_cleaner_latency   0
zk_avg_write_final_proc_time_ms 1.0
zk_min_write_final_proc_time_ms 0
zk_max_write_final_proc_time_ms 5
zk_cnt_write_final_proc_time_ms 5
zk_sum_write_final_proc_time_ms 5
zk_p50_write_final_proc_time_ms 0
zk_p95_write_final_proc_time_ms 5
zk_p99_write_final_proc_time_ms 5
zk_p999_write_final_proc_time_ms        5
zk_avg_proposal_ack_creation_latency    4.6
zk_min_proposal_ack_creation_latency    2
zk_max_proposal_ack_creation_latency    13
zk_cnt_proposal_ack_creation_latency    5
zk_sum_proposal_ack_creation_latency    23
zk_p50_proposal_ack_creation_latency    3
zk_p95_proposal_ack_creation_latency    13
zk_p99_proposal_ack_creation_latency    13
zk_p999_proposal_ack_creation_latency   13
zk_avg_proposal_latency 2.2
zk_min_proposal_latency 2
zk_max_proposal_latency 3
zk_cnt_proposal_latency 5
zk_sum_proposal_latency 11
zk_p50_proposal_latency 2
zk_p95_proposal_latency 3
zk_p99_proposal_latency 3
zk_p999_proposal_latency        3
zk_avg_om_proposal_process_time_ms      0.0
zk_min_om_proposal_process_time_ms      0
zk_max_om_proposal_process_time_ms      0
zk_cnt_om_proposal_process_time_ms      0
zk_sum_om_proposal_process_time_ms      0
zk_p50_om_proposal_process_time_ms      0
zk_p95_om_proposal_process_time_ms      0
zk_p99_om_proposal_process_time_ms      0
zk_p999_om_proposal_process_time_ms     0
zk_avg_sync_processor_queue_and_flush_time_ms   2.8
zk_min_sync_processor_queue_and_flush_time_ms   0
zk_max_sync_processor_queue_and_flush_time_ms   11
zk_cnt_sync_processor_queue_and_flush_time_ms   5
zk_sum_sync_processor_queue_and_flush_time_ms   14
zk_p50_sync_processor_queue_and_flush_time_ms   1
zk_p95_sync_processor_queue_and_flush_time_ms   11
zk_p99_sync_processor_queue_and_flush_time_ms   11
zk_p999_sync_processor_queue_and_flush_time_ms  11
zk_avg_propagation_latency      5.8
zk_min_propagation_latency      4
zk_max_propagation_latency      12
zk_cnt_propagation_latency      5
zk_sum_propagation_latency      29
zk_p50_propagation_latency      4
zk_p95_propagation_latency      12
zk_p99_propagation_latency      12
zk_p999_propagation_latency     12
zk_avg_server_write_committed_time_ms   0.5
zk_min_server_write_committed_time_ms   0
zk_max_server_write_committed_time_ms   1
zk_cnt_server_write_committed_time_ms   4
zk_sum_server_write_committed_time_ms   2
zk_p50_server_write_committed_time_ms   1
zk_p95_server_write_committed_time_ms   1
zk_p99_server_write_committed_time_ms   1
zk_p999_server_write_committed_time_ms  1
zk_avg_sync_processor_queue_time_ms     0.0
zk_min_sync_processor_queue_time_ms     0
zk_max_sync_processor_queue_time_ms     0
zk_cnt_sync_processor_queue_time_ms     5
zk_sum_sync_processor_queue_time_ms     0
zk_p50_sync_processor_queue_time_ms     0
zk_p95_sync_processor_queue_time_ms     0
zk_p99_sync_processor_queue_time_ms     0
zk_p999_sync_processor_queue_time_ms    0
zk_avg_sync_processor_queue_flush_time_ms       2.6
zk_min_sync_processor_queue_flush_time_ms       0
zk_max_sync_processor_queue_flush_time_ms       10
zk_cnt_sync_processor_queue_flush_time_ms       5
zk_sum_sync_processor_queue_flush_time_ms       13
zk_p50_sync_processor_queue_flush_time_ms       1
zk_p95_sync_processor_queue_flush_time_ms       10
zk_p99_sync_processor_queue_flush_time_ms       10
zk_p999_sync_processor_queue_flush_time_ms      10
zk_avg_write_commitproc_time_ms 9.0
zk_min_write_commitproc_time_ms 9
zk_max_write_commitproc_time_ms 9
zk_cnt_write_commitproc_time_ms 1
zk_sum_write_commitproc_time_ms 9
zk_p50_write_commitproc_time_ms 9
zk_p95_write_commitproc_time_ms 9
zk_p99_write_commitproc_time_ms 9
zk_p999_write_commitproc_time_ms        9
zk_avg_app_read_per_namespace   72.0
zk_min_app_read_per_namespace   72
zk_max_app_read_per_namespace   72
zk_cnt_app_read_per_namespace   1
zk_sum_app_read_per_namespace   72
zk_avg_app_write_per_namespace  8.5
zk_min_app_write_per_namespace  4
zk_max_app_write_per_namespace  15
zk_cnt_app_write_per_namespace  8
zk_sum_app_write_per_namespace  68
zk_avg_zookeeper_write_per_namespace    188.0
zk_min_zookeeper_write_per_namespace    188
zk_max_zookeeper_write_per_namespace    188
zk_cnt_zookeeper_write_per_namespace    2
zk_sum_zookeeper_write_per_namespace    376
zk_avg_application_write_per_namespace  16.0
zk_min_application_write_per_namespace  12
zk_max_application_write_per_namespace  22
zk_cnt_application_write_per_namespace  4
zk_sum_application_write_per_namespace  64

Check if server is running in read-only mode.

$ echo isro | nc zookeeper3.example.org 2181; echo
rw

I have omitted some options like gtmk, stmk as tracing deserves its own blog post.