kafka 常用命令

create topic

kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 3 --partitions 1 --topic egg-test

list the topic from the kafka cluster

$kafka-topics --list --bootstrap-server localhost:9092

write/produce the message into kafka for the topic

kafka-console-producer.sh --broker-list localhost:9092 --topic egg-test
OR
kafka-console-producer.sh --bootstrap-server localhost:9092 --topic test

$kafka-console-producer.sh --broker-list localhost:9092 --topic egg-test
2020-07-14T10:28:40.063+0000 INFO Kafka version: 2.2.2 (org.apache.kafka.common.utils.AppInfoParser)
2020-07-14T10:28:40.063+0000 INFO Kafka commitId: 1d348535a0a747d1 (org.apache.kafka.common.utils.AppInfoParser)
2020-07-14T10:28:40.254+0000 INFO Cluster ID: Egm3skQ0R_yMdOZjbM_6rw (org.apache.kafka.clients.Metadata)
hello world           
>

consume/check the message in console for the topic

bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic egg-test --from-beginning

check the description for the topic

$kafka-topics --describe  --bootstrap-server localhost:9092 --topic faultIndication
...
2020-07-14T09:56:19.371+0000 INFO Kafka version: 2.2.2 (org.apache.kafka.common.utils.AppInfoParser)
2020-07-14T09:56:19.371+0000 INFO Kafka commitId: 1d348535a0a747d1 (org.apache.kafka.common.utils.AppInfoParser)
Topic:faultIndication    PartitionCount:3    ReplicationFactor:3 Configs:min.insync.replicas=2,cleanup.policy=delete,segment.bytes=10485760,min.compaction.lag.ms=0,retention.bytes=104857600
    Topic: AdpFaultIndication   Partition: 0    Leader: 2   Replicas: 2,0,1 Isr: 2,0,1
    Topic: AdpFaultIndication   Partition: 1    Leader: 1   Replicas: 1,2,0 Isr: 1,2,0
    Topic: AdpFaultIndication   Partition: 2    Leader: 0   Replicas: 0,1,2 Isr: 0,1,2

fetch 最多10条messages

kafka-console-consumer --bootstrap-server localhost:9092 --topic my-topic  --max-messages 10

查看message 在partition 中的分布

kafka-run-class.sh kafka.tools.GetOffsetShell   --broker-list localhost:9092   --topic my-topic   --time -1

## output example:
my-topic:0:10023
my-topic:1:9821
my-topic:2:10567

check kafka version

bash-4.4$ kafka-topics.sh --version
3.5.1 (Commit:2c6fb6c54472e90a)

查看 kafka cluster broker members

kafka-broker-api-versions.sh --bootstrap-server localhost:9092

可以查看partition 在每个broker 上的分布。

kafka-topics.sh --bootstrap-server localhost:9092 --describe

# output example 如下, 其中 0,1,2 是指borker 的ID.
# Topic: my-topic       Partition: 6    Leader: 0       Replicas: 0,1,2 Isr: 0,1,2
# Topic: my-topic       Partition: 7    Leader: 2       Replicas: 2,0,1 Isr: 2,0,1
# Topic: my-topic       Partition: 8    Leader: 1       Replicas: 1,2,0 Isr: 1,2,0

# 所以可以通过 如下命令可以知道某个broker上有那些partition

kafka-topics.sh --describe --bootstrap-server localhost:9092 | grep -E 'Leader: 2'

查看特定的topic 的配置和partition 情况

kafka-topics --bootstrap-server localhost:9092 --describe --topic my-topic

## Output looks like below

    Topic: my-topic       
    TopicId: BICkLVBYSIyjsSf4roqouQ 
    PartitionCount: 180     
    ReplicationFactor: 3    
    Configs: min.insync.replicas=2,cleanup.policy=delete,segment.bytes=10485760,min.compaction.lag.ms=0,retention.bytes=345985024
        
    Topic: my-topic       Partition: 0    Leader: 1       Replicas: 1,0,2 Isr: 1,0,2
    Topic: my-topic       Partition: 1    Leader: 0       Replicas: 0,2,1 Isr: 0,2,1
    ...

获取cluster ID

# 登录zookeeper 查看kafka 的cluster ID
[zk: localhost:2181(CONNECTED) 5] get  /my-test-message-bus/cluster/id
{"version":"1","id":"B807omxbRP2tSBN4foce8w"}


# 通过 kafka 命令
bash-4.4$ kafka-cluster.sh cluster-id --bootstrap-server localhost:9092
Cluster ID: B807omxbRP2tSBN4foce8w

查看server配置

# --broker 1 是指查看broker ID 为 1 的broker 配置。不过这些配置在所有broker 上都是一样的，所以也相当于查看cluster 的配置了。

bash-4.4$ kafka-configs --bootstrap-server localhost:9092 --broker 1 --all --describe | grep leader
  auto.leader.rebalance.enable=true sensitive=false synonyms={DEFAULT_CONFIG:auto.leader.rebalance.enable=true}
  leader.imbalance.check.interval.seconds=300 sensitive=false synonyms={DEFAULT_CONFIG:leader.imbalance.check.interval.seconds=300}
  leader.imbalance.per.broker.percentage=10 sensitive=false synonyms={DEFAULT_CONFIG:leader.imbalance.per.broker.percentage=10}
  unclean.leader.election.enable=false sensitive=false synonyms={DEFAULT_CONFIG:unclean.leader.election.enable=false}