kafka_2.11-0.10.2.1 的生產者 消費者的示例（new producer api）

Jacy_Wang發表於2019-01-15
KafkaAPI

環境，以及單獨的pom.xml檔案

環境：java 1.8 ，kafka_2.11-0.10.2.1

pom.xml檔案如下

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
 
    <groupId>kafka_demo</groupId>
    <artifactId>kafka_demo</artifactId>
    <version>1.0-SNAPSHOT</version>
 
    <properties>
        <kafka.version>0.10.2.1</kafka.version>
    </properties>
 
    <dependencies>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka_2.11</artifactId>
            <version>${kafka.version}</version>
        </dependency>
    </dependencies>
 
 
</project>

1、生產者程式碼

package p_c_demo1;
 
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import utils.KafkaProperties;
 
import java.util.Properties;
import java.util.concurrent.ExecutionException;
 
/**
 *
 */
public class Producer_demo1 extends Thread{
    public static void main(String[] args) {
 
        //boolean isAsync = args.length == 0 || !args[0].trim().equalsIgnoreCase("sync");
        boolean isAsync = true;
        Producer_demo1 producerThread = new Producer_demo1(KafkaProperties.INTOPIC, isAsync);
        producerThread.start();
    }
    private final KafkaProducer<Integer, String> producer;
    private final String topic;
    private final Boolean isAsync;
 
    public Producer_demo1(String topic, Boolean isAsync) {
        Properties props = new Properties();
        props.put("bootstrap.servers",
                KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT0
                        + "," + KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT1
                        + "," + KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT2);
        props.put("client.id", "Producer_demo1");
 
        //開始的時候下面5個引數未設定，導致消費時取不到資料，需要注意
       /* acks=0時，producer不會等待確認，直接新增到socket等待傳送；
        acks=1時，等待leader寫到local log就行；
        acks=all或acks=-1時，等待isr中所有副本確認
        */
        props.put("acks", "all");
        //傳送失敗重試
        props.put("retries", 0);
        //批次傳送，不會嘗試大於此值的容量
        props.put("batch.size", 16384);
        //預設設定為0，
        // 具體引數參考：http://kafka.apache.org/0102/documentation.html#producerconfigs
        props.put("linger.ms", 1);
 
        props.put("buffer.memory", 33554432);
 
        props.put("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
 
        producer = new KafkaProducer<Integer, String>(props);
        //方法傳進來的引數
        this.topic = topic;
        this.isAsync = isAsync;
    }
    
    //持續保持資料傳送
    public void run() {
        System.out.println("ProducerThread--run");
        int messageNo = 1;
        //這裡面應該是自己的資料邏輯處理
        while (true) {
            String messageStr = "Message_" + messageNo;
            long startTime = System.currentTimeMillis();
            /*
            檢視原始碼可以知道第二個send其實就是呼叫的第一個send 但是callback為null
            send
            public Future<RecordMetadata> send(ProducerRecord<K, V> record) {
                return this.send(record, (Callback)null);
    }
             */
            if (isAsync) { // Send asynchronously
                producer.send(
                        new ProducerRecord<Integer, String>(topic, messageNo, messageStr),
                        new DemoCallBack(startTime, messageNo, messageStr));
            } else { // Send synchronously
 
                try {
                    producer.send(new ProducerRecord<Integer, String>(topic,messageNo,messageStr)).get();
                } catch (InterruptedException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (ExecutionException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
                System.out.println("Sent message: (" + messageNo + ", " + messageStr + ")");
 
            }
            //messageNo;
            System.out.println("Sent message: (" + messageNo++ + ", " + messageStr + ")");
            //休息0.5秒
            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
 
            if(messageNo==5000){
 
                //break;
            }
        }
    }
}
 
class DemoCallBack implements Callback {
 
    private final long startTime;
    private final int key;
    private final String message;
 
    public DemoCallBack(long startTime, int key, String message) {
        this.startTime = startTime;
        this.key = key;
        this.message = message;
    }
 
    /**
     * A callback method the user can implement to provide asynchronous handling of request completion. This method will
     * be called when the record sent to the server has been acknowledged. Exactly one of the arguments will be
     * non-null.
     *
     * @param metadata  The metadata for the record that was sent (i.e. the partition and offset). Null if an error
     *                  occurred.
     * @param exception The exception thrown during processing of this record. Null if no error occurred.
     */
    public void onCompletion(RecordMetadata metadata, Exception exception) {
        long elapsedTime = System.currentTimeMillis() - startTime;
        if (metadata != null) {
            System.out.println(
                    "message(" + key + ", " + message + ") sent to partition(" + metadata.partition() +
                            "), " +
                            "offset(" + metadata.offset() + ") in " + elapsedTime + " ms");
        } else {
            exception.printStackTrace();
        }
    }
 
}

2、消費者程式碼

package consumerDemo;
 
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import utils.KafkaProperties;
 
import java.util.Arrays;
import java.util.Properties;
 
public class Conumer_demo1 extends Thread{
 
    public static void main(String[] args) {
        p_c_demo1.Conumer_demo1 consumerThread = new p_c_demo1.Conumer_demo1(KafkaProperties.INTOPIC);
        consumerThread.start();
    }
 
    private final KafkaConsumer<Integer, String> consumer;
 
    private final String topic;
    private static final Logger LOG = LoggerFactory.getLogger(Conumer_demo1.class);
    public Conumer_demo1(String topic) {
 
        Properties props = new Properties();
        //bootstrap.servers   必要
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
                KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT0
        + "," + KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT1
        + "," + KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT2);
        //group id
        props.put(ConsumerConfig.GROUP_ID_CONFIG, "producer-consumer-demo1");
        //是否後臺自動提交offset 到kafka
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
        //消費者偏移自動提交到Kafka的頻率（以毫秒為單位enable.auto.commit）設定為true
        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
        //故障檢測，心跳檢測機制 的間隔時間，，在該值範圍內，沒有接收到心跳，則會刪除該消費者
        //並啟動再平衡（rebanlance）,值必須在group.min.session.timeout 和 group.max.session.timeout.ms之間
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
        //key - value 的序列化類
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.IntegerDeserializer");
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
 
        this.consumer = new KafkaConsumer<Integer, String>(props);
        this.topic = topic;
    }
    public void run() {
        System.out.println("ConsumerThread--run");
 
        consumer.subscribe(Arrays.asList(KafkaProperties.INTOPIC));
 
        // consumer.subscribe(Collections.singletonList(this.topic));
        while (true) {
 
            //consumer.poll()
            ConsumerRecords<Integer, String> records = consumer.poll(200);
 
            for (ConsumerRecord<Integer, String> record : records) {
                System.out.println("Received message: (" + record.key() + ", " + record.value()
                        + ") offset " + record.offset()
                        + " partition " + record.partition() + ")");
            }
        }
    }
 
}

3、還有一個工具類，放我們的各項引數設定

package utils;
 
public class KafkaProperties {
    public static final String INTOPIC = "producer_consumer_demo1";
    //public static final String OUTTOPIC = "topic2";
    public static final String KAFKA_SERVER_URL = "make.spark.com";
    public static final int KAFKA_SERVER_PORT0 = 9092;
    public static final int KAFKA_SERVER_PORT1 = 9093;
    public static final int KAFKA_SERVER_PORT2 = 9094;
    public static final int KAFKA_PRODUCER_BUFFER_SIZE = 65536;
    public static final int CONNECTION_TIMEOUT = 100000;
    public static final String CLIENT_ID = "SimpleConsumerDemoClient";
 
    private KafkaProperties() {}
 
}

這個時候啟動我們，生產者，就會開始生產資料，同時執行我麼你的消費者 就可以看到我們的消費資訊，具體在哪個分割槽，消費到那個偏移量，如果同時多開，兩個消費者，可以看到rebalane的機制，會重新再平衡每個消費者，消費的分割槽，這裡的前提是，要修改你的topic，改成多個分割槽才可以，生產者預設建立的topic是隻有一個分割槽的。

原文：https://blog.csdn.net/maketubu7/article/details/81535087

相關文章