美團Cat原始碼淺析（四）服務端訊息分發

本篇將從TcpSocketReceiver接收訊息、解碼消費開始來分析Cat服務端訊息分發。
備註：
Cat版本：v3.0.0
github：https://github.com/dianping/cat

public final class TcpSocketReceiver implements LogEnabled {
    public void init() {
        try {
            startServer(m_port);
        } catch (Throwable e) {
            m_logger.error(e.getMessage(), e);
        }
    }

    public synchronized void startServer(int port) throws InterruptedException {
        boolean linux = getOSMatches("Linux") || getOSMatches("LINUX");
        int threads = 24;
        ServerBootstrap bootstrap = new ServerBootstrap();

        m_bossGroup = linux ? new EpollEventLoopGroup(threads) : new NioEventLoopGroup(threads);
        m_workerGroup = linux ? new EpollEventLoopGroup(threads) : new NioEventLoopGroup(threads);
        bootstrap.group(m_bossGroup, m_workerGroup);
        bootstrap.channel(linux ? EpollServerSocketChannel.class : NioServerSocketChannel.class);

        bootstrap.childHandler(new ChannelInitializer<SocketChannel>() {
            @Override
            protected void initChannel(SocketChannel ch) throws Exception {
                ChannelPipeline pipeline = ch.pipeline();
                pipeline.addLast("decode", new MessageDecoder());
                pipeline.addLast("encode", new ClientMessageEncoder());
            }
        });

        bootstrap.childOption(ChannelOption.SO_REUSEADDR, true);
        bootstrap.childOption(ChannelOption.TCP_NODELAY, true);
        bootstrap.childOption(ChannelOption.SO_KEEPALIVE, true);
        bootstrap.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);

        try {
            m_future = bootstrap.bind(port).sync();
            m_logger.info("start netty server!");
        } catch (Exception e) {
            m_logger.error("Started Netty Server Failed:" + port, e);
        }
    }
}

MessageDecoder的祖先類實現了ChannelInboundHandler介面，是網路IO事件具體處理類，當客戶端將日誌資料上傳到伺服器之後，會交給MessageDecoder 解碼資料，然後進行後續處理。

class MessageDecoder {
    @Inject
    private MessageHandler m_handler;
    protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List<Object> out) {
        int length = buffer.readInt();
        ByteBuf readBytes = buffer.readBytes(length + 4);
        DefaultMessageTree tree = (DefaultMessageTree) CodecHandler.decode(readBytes);
        tree.setBuffer(readBytes);
        m_handler.handle(tree);
    }
}
class DefaultMessageHandler {
    public void handle(MessageTree tree) {
        if (m_consumer == null) {
            m_consumer = lookup(MessageConsumer.class);
        }
        m_consumer.consume(tree);
    }
}

訊息解碼完成後最終呼叫MessageConsumer.consume方法傳遞給消費方。

class RealtimeConsumer {
    @Inject
    private MessageAnalyzerManager m_analyzerManager;
    @Override
    public void consume(MessageTree tree) {
        long timestamp = tree.getMessage().getTimestamp();
        //找到對應週期
        Period period = m_periodManager.findPeriod(timestamp);
        if (period != null) {
            //分發
            period.distribute(tree);
        } else {
            m_serverStateManager.addNetworkTimeError(1);
        }
    }
    @Override
    public void initialize() throws InitializationException {
        //初始化週期管理器
        m_periodManager = new PeriodManager(HOUR, m_analyzerManager, m_serverStateManager, m_logger);
        m_periodManager.init();
        Threads.forGroup("cat").start(m_periodManager);
    }
}

consumer()根據訊息的時間戳找到對應的週期開始訊息的分發。以下為訊息分發流程圖：

從圖上可以看出：
1、每個週期為整點開始的一個小時，一個週期對應一系列週期任務；
2、每個週期任務對應一個執行緒，週期任務、執行緒、分析器一一對應；
3、每個類別的分析器數量可配，TransactionAnalyzer預設兩個分析器。
首先看注入的分析管理器MessageAnalyzerManager的初始化邏輯：

class DefaultMessageAnalyzerManager {
    //存放每個週期對應的分析器
    private Map<Long, Map<String, List<MessageAnalyzer>>> m_analyzers = new HashMap<Long, Map<String, List<MessageAnalyzer>>>();
    //根據週期時間和分析器類別，獲取當前週期、該類別分析器所有例項
    @Override
    public List<MessageAnalyzer> getAnalyzer(String name, long startTime) {
        、、、
        //獲取當前週期分析器
        Map<String, List<MessageAnalyzer>> map = m_analyzers.get(startTime);
        if (map == null) {
            synchronized (m_analyzers) {
                map = m_analyzers.get(startTime);
                if (map == null) {
                    map = new HashMap<String, List<MessageAnalyzer>>();
                    m_analyzers.put(startTime, map);
                }
            }
        }
        //該類別分析器所有例項
        List<MessageAnalyzer> analyzers = map.get(name);

        if (analyzers == null) {
            synchronized (map) {
                analyzers = map.get(name);
                if (analyzers == null) {
                    analyzers = new ArrayList<MessageAnalyzer>();
                    MessageAnalyzer analyzer = lookup(MessageAnalyzer.class, name);
                    analyzer.setIndex(0);
                    analyzer.initialize(startTime, m_duration, m_extraTime);
                    analyzers.add(analyzer);
                    //該分析器的例項數（除了TransactionAnalyzer預設為2，其他都是1）
                    int count = analyzer.getAnanlyzerCount(name);
                    for (int i = 1; i < count; i++) {
                        MessageAnalyzer tempAnalyzer = lookup(MessageAnalyzer.class, name);
                        tempAnalyzer.setIndex(i);
                        tempAnalyzer.initialize(startTime, m_duration, m_extraTime);
                        analyzers.add(tempAnalyzer);
                    }
                    map.put(name, analyzers);
                }
            }
        }
        return analyzers;
        、、、
    }
    //所有有效的分析器
    @Override
    public void initialize() throws InitializationException {
        Map<String, MessageAnalyzer> map = lookupMap(MessageAnalyzer.class);
        m_analyzerNames = new ArrayList<String>(map.keySet());
        、、、
        //移除禁用的分析器
        、、、
    }
}

接下來來看initialize()方法中週期管理器的初始化邏輯。

class PeriodManager {
    public void init() {
        long startTime = m_strategy.next(System.currentTimeMillis());
        startPeriod(startTime);
    }

    private void startPeriod(long startTime) {
        long endTime = startTime + m_strategy.getDuration();
        Period period = new Period(startTime, endTime, m_analyzerManager, m_serverStateManager, m_logger);
        m_periods.add(period);
//啟動一個週期的所有周期任務
        period.start();
    }
}

startPeriod(）初始化當前週期，並啟動當前週期的所有周期任務。

class Period {
    public Period(long startTime, long endTime, MessageAnalyzerManager analyzerManager,
                  ServerStatisticManager serverStateManager, Logger logger) {
        //每一種類別的分析器，都會有至少一個MessageAnalyzer的例項，每個MessageAnalyzer都由一個對應的PeriodTask來分配任務，MessageAnalyzer與PeriodTask是1對1的關係
        List<MessageAnalyzer> messageAnalyzers = m_analyzerManager.getAnalyzer(name, startTime);
        for (MessageAnalyzer analyzer : messageAnalyzers) {
            //佇列大小30000
            MessageQueue queue = new DefaultMessageQueue(QUEUE_SIZE);
            PeriodTask task = new PeriodTask(analyzer, queue, startTime);
            List<PeriodTask> analyzerTasks = m_tasks.get(name);
            if (analyzerTasks == null) {
                analyzerTasks = new ArrayList<PeriodTask>();
                m_tasks.put(name, analyzerTasks);
            }
            analyzerTasks.add(task);
        }
    }
    //啟動一個週期的所有周期任務
    public void start() {
        for (Entry<String, List<PeriodTask>> tasks : m_tasks.entrySet()) {
            List<PeriodTask> taskList = tasks.getValue();
            for (int i = 0; i < taskList.size(); i++) {
                PeriodTask task = taskList.get(i);
                task.setIndex(i);
                Threads.forGroup("Cat-RealtimeConsumer").start(task);
            }
        }
    }
}

最後回過頭來看RealTimeConsumer.consume()方法中Period.distribute(messageTree)訊息分發邏輯：

    public void distribute(MessageTree tree) {
        for (Entry<String, List<PeriodTask>> entry : m_tasks.entrySet()) {
            List<PeriodTask> tasks = entry.getValue();
            int length = tasks.size();
            int index = 0;
            boolean manyTasks = length > 1;
            if (manyTasks) {
                index = Math.abs(domain.hashCode()) % length;
            }
            PeriodTask task = tasks.get(index);
            boolean enqueue = task.enqueue(tree);
            if (!enqueue) {
                if (manyTasks) {
                    task = tasks.get((index + 1) % length);
                    enqueue = task.enqueue(tree);
                    if (!enqueue) {
                        success = false;
                    }
                } else {
                    success = false;
                }
            }
        }
    }

遍歷當前週期的所有周期任務，非同步入隊。