kafka-Java-SpringBoot-consumerAPI開發

ConsumerAPI的開發邏輯和Product是一樣的,只不過多了一項必填選項group_id.
屬性:

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.util.List;

/**
 * @Author dw07-Riven770[wudonghua@gznb.com]
 * @Date 2017/12/1315:58
 */
@ConfigurationProperties(prefix = "Riven.kafka.consumer")
public class ConsumerConfiguration {
    //kafka伺服器列表
    private String bootstrapServers;

    /**
     * 如果設定成true,偏移量由auto.commit.interval.ms控制自動提交的頻率。
     * <p>
     * 如果設定成false,不需要定時的提交offset，可以自己控制offset，當訊息認為已消費過了，這個時候再去提交它們的偏移量。
     * 這個很有用的，當消費的訊息結合了一些處理邏輯，這個訊息就不應該認為是已經消費的，直到它完成了整個處理。
     */
    private Boolean enableAutoCommit = false;

    /**
     * 提交延遲毫秒數
     */
    private int autoCommitIntervalMs = 100;

    /**
     * 執行超時時間
     */
    private int sessionTimeoutMs = 15000;

    /**
     * 每次最少拉取多少資料
     */
    private int fetchMinBytes = 1;

    /**
     * 在單次呼叫中的最大返回
     */
    private int maxPollRecords = 300;

    /**
     * 該Consumer屬於的組
     */
    private String groupId ;

    /**
     * 在consumter端配置檔案中(或者是ConsumerConfig類引數)有個"autooffset.reset"(在kafka 0.8版本中為auto.offset.reset),
     * 有2個合法的值"largest"/"smallest",預設為"largest",此配置參數列示當此groupId下的消費者,在ZK中沒有offset值時(比如新的groupId,或者是zk資料被清空),
     * consumer應該從哪個offset開始消費.largest表示接受接收最大的offset(即最新訊息),smallest表示最小offset,即從topic的開始位置消費所有訊息.
     */
    private String autoOffseReset = "latest";

    /**
     * 同一個組下 啟動幾個consumer來獲取kafka的訊息
     */
    private int consumerAmount = 3;

    /**
     * 設定啟動的consumer多久超時
     */
    private int pollTimeout = 5000;

    private List<String> topics;

    private String keySerializer = StringDeserializer.class.getName();
    private String valueSerializer = StringDeserializer.class.getName();

}

配置類:

import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import riven.kafka.api.configuration.ConsumerConfiguration;

import java.util.HashMap;
import java.util.Map;

/**
 * @Author dw07-Riven770[wudonghua@gznb.com]
 * @Date 2017/12/1411:16
 * 配置Consumer選項
 * 初始化consumer_S
 */
@Configuration
@EnableKafka
@EnableConfigurationProperties(ConsumerConfiguration.class)
@ConditionalOnProperty(name = {"Riven.kafka.consumer.bootstrapServers", "Riven.kafka.consumer.groupId"}, matchIfMissing = false)
public class ConsumerInitialize {


    private Logger logger = LoggerFactory.getLogger(this.getClass());

    /**
     * 初始化引數
     *
     * @param config
     * @return
     */
    private Map<String, Object> assembleProducer(ConsumerConfiguration config) {
        Map<String, Object> propsMap = new HashMap<>();
        if (StringUtils.isBlank(config.getBootstrapServers()))
            throw new RuntimeException("缺失kafka叢集列表,初始化失敗");
        propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers());

        propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, config.getEnableAutoCommit());
        //提交延遲毫秒數
        propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, config.getAutoCommitIntervalMs());
        //執行超時時間
        propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, config.getSessionTimeoutMs());
        propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, config.getKeySerializer());
        propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, config.getValueSerializer());
        propsMap.put(ConsumerConfig.FETCH_MIN_BYTES_CONFIG, config.getFetchMinBytes());
        propsMap.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, config.getMaxPollRecords());
        //組ID
        if (StringUtils.isBlank(config.getGroupId()))
            throw new RuntimeException("缺失Consumer組資訊,初始化失敗");
        propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, config.getGroupId());

        propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, config.getAutoOffseReset());
        return propsMap;
    }

    @Bean
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory
            (ConsumerConfiguration ver) {
        ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        try {
            factory.setConsumerFactory(consumerFactory(ver));
            factory.setConcurrency(ver.getConsumerAmount());//啟動的consumer個數
            factory.getContainerProperties().setPollTimeout(ver.getPollTimeout());//consumer;連線超時時間ms
            logger.info("初始化Consumer_S完成,共啟動 {} 個Consumer", ver.getConsumerAmount());
        } catch (Exception e) {
            logger.info("初始化Consumer_S失敗!");
            e.printStackTrace();
        }
        return factory;
    }

    @org.jetbrains.annotations.NotNull
    private ConsumerFactory<String, String> consumerFactory(ConsumerConfiguration ver) {
        return new DefaultKafkaConsumerFactory<>(assembleProducer(ver));
    }

最後,在配置檔案根目錄下建立Spring監聽器:
spring.factories檔案
並新增需要Spring監聽初始化的類路徑(多個使用,逗號隔開):

org.springframework.boot.autoconfigure.EnableAutoConfiguration=riven.kafka.api.producer.ProducerInitialize,riven.kafka.api.consumer.ConsumerInitialize