Python使用Kafka处理数据的方法详解

pip install kafka-python

from kafka import KafkaProducer

producer = KafkaProducer(bootstrap_servers=['localhost:9092'])

producer.send('test', b'Hello, Kafka!')

from kafka import KafkaConsumer

consumer = KafkaConsumer('test', bootstrap_servers=['localhost:9092'])

for message in consumer:
    print(message.value)

from kafka import KafkaProducer, KafkaConsumer
from kafka.errors import KafkaError

producer = KafkaProducer(bootstrap_servers=['localhost:9092'])

for i in range(10):
    message = 'Message {}'.format(i)
    future = producer.send('test', bytes(message, 'utf-8'))
    try:
        record_metadata = future.get(timeout=10)
        print('Message {} sent to partition {} with offset {}'.format(message, record_metadata.partition, record_metadata.offset))
    except KafkaError as e:
        print('Failed to send message {}: {}'.format(message, e))

consumer = KafkaConsumer('test', bootstrap_servers=['localhost:9092'], auto_offset_reset='earliest', enable_auto_commit=True, group_id='my-group', max_poll_records=10)

while True:
    messages = consumer.poll(timeout_ms=1000)
    if not messages:
        continue
    for topic_partition, records in messages.items():
        for record in records:
            print(record.value.decode('utf-8'))