Kafka學習之（四）PHP操作Kafka

OldBoy~發表於2018-01-17

簡單測試

環境：Centos6.4，PHP7，kafka伺服器IP：192.168.9.154，PHP伺服器:192.168.9.157

在192.168.9.157建立目錄和檔案。

//生產者
<?php
require './modules/kafka.php';
$rk = new kafka();
$rk->send(['hello my kafka']);
echo 'OK~';

//消費者
<?php
require './modules/kafka.php';
$rk = new kafka();
$rk->consumer();

//Kafka類
<?php
class kafka
{
    public $broker_list = '192.168.9.154:9092';  //現在是一個，也可以多個，用逗號隔開
    public $topic = 'mytopic';                      //定義topic
    public $partition = 0;                         //定義topic的物理分組，這裡是0
    public $logFile = './kafkalog/info.log';     //計劃在消費的時候寫日誌，指定日誌檔案

    protected $producer = null;                  //儲存producer物件
    protected $consumer = null;                     //儲存consumer物件

    public function __construct()
    {

        if (empty($this->broker_list)) 
        {
            echo 'broker not config';
        }
        $rk = new \Rdkafka\Producer();  //例項化物件

        if (empty($rk)) {
            echo 'producer error1';
        }
        $rk->setLogLevel(LOG_DEBUG);  //設定錯誤級別
        if(!$rk->addBrokers($this->broker_list)) {//設定Broker地址
            echo 'producer error2';
        }     
        $this->producer = $rk;
    }

    //生產者的方法（生產者把日誌向訊息佇列傳送）
    public function send($message = []) 
    {
        
        $topic = $this->producer->newTopic($this->topic);  //建立topic

        $topic->produce(RD_KAFKA_PARTITION_UA, $this->partition, json_encode([$message]); //生產
    }
    //消費者方法 （監聽訊息佇列）
    public function consumer()
    {
        $conf = new \Rdkafka\Conf();
        $conf->set('group.id', 0);
        $conf->set('metadata.broker.list', $this->broker_list);
        $topicConf = new \Rdkafka\topicConf();
        $topicConf->set('auto.offset.reset', 'smallest');

        $conf->setDefaultTopicConf($topicConf);

        $consumer = new \Rdkafka\kafkaConsumer($conf);

        $consumer->subscribe([$this->topic]); //訂閱

        echo "wating for message....\n";

        while(true) {
            $message = $consumer->consume(0*1000);
            switch ($message->err) {
                case RD_KAFKA_RESP_ERR_NO_ERROR:
                    echo '要處理訊息了~~~';
                    $messageInfo = $message->payload;
                    // echo $messageInfo."\n";
                    break;
            }
            sleep(1);
        }
    }
}

記住消費者PHP檔案要在終端執行:php consumer.php。

這裡就不測試了。

工作程式碼

/**
     * 將使用者的登陸資訊放到 Kafka
     *
     */
    public function sendCustomerLoginInfoToKafka($param){
        $customerLoginInfoServiceClient = new CustomerLoginInfoServiceClient();
        $msg =  json_encode($param);
        $topic=isset(Yii::app()->params['customer_login_info_topic'])?Yii::app()->params['customer_login_info_topic']:'e_user_login_info';
        $result = $customerLoginInfoServiceClient->add($topic, $msg);
    }

/**
 * 客戶登陸資訊 服務化介面呼叫client端
 */
class CustomerLoginInfoServiceClient {

    public function add($topic, $msg) {
        //直接進kafka不再呼叫java服務
        EdjKafka::getInstance()->putin(array("topic" => $topic, "payload" => $msg));
    }

}

class EdjKafka {

    private static $instance;

    public static function getInstance($className=__CLASS__) {
        if (empty(self::$instance)) {
            self::$instance = new $className();
        }
        return self::$instance;
    }

    public function putin($params) {
        $task = array(
            'class' => __CLASS__,
            'method' => 'putin_job',
            'params' => $params,
        );
        Queue::model()->putin($task, 'phptokafka_0000');
    }

    public function putin_job($params) {
        KafkaProducer::getInstance()->putin($params["topic"], $params["payload"]);
    }
}

<?php
require_once(Yii::app()->basePath.'/vendors/kafka/autoload.php');  //kafka包在最下面

class KafkaProducer {

    private static $instance;
    private $producer;
    private $partitionCountMap = array();

    public static function getInstance($className=__CLASS__) {
        if (empty(self::$instance)) {
            self::$instance = new $className();
        }
        return self::$instance;
    }

    public function __construct() {
        $brokers = Yii::app()->params['kafka']['brokers'];

        $newProducer = \Kafka\Produce::getInstance($brokers, 3000, $brokers);
        $newProducer->setRequireAck(-1);
        $this->producer = $newProducer;
    }

    private function getPartitionCount($topic, $force=false) {
        $now = time();

        //3分鐘查詢一次patition
        if( !$force && array_key_exists($topic, $this->partitionCountMap) && $this->partitionCountMap[$topic]["expire"] > $now ) {
            return $this->partitionCountMap[$topic]["count"];
        }


        //獲取到topic下可用的partitions
        $this->producer->getClient()->refreshMetadata();
        $partitions = $this->producer->getAvailablePartitions($topic);
        EdjLog::info(__METHOD__.'|'.$topic.'|get partition|'.json_encode($partitions));
        $partitionCount = count($partitions);
        if ($partitionCount == 0) {
            EdjLog::error(__METHOD__."|".$topic."|topic partitions count 0");
        }

        $this->partitionCountMap[$topic] = array("count" => $partitionCount, "expire" => $now + 180);

        return $partitionCount;
    }

    public function putin($topic, $payload) {
        if(empty($topic)) {
            return;
        }

        $partitionCount = $this->getPartitionCount($topic);

        if ($partitionCount != 0) {
            try {
                $pid = time() % $partitionCount;
                $this->producer->setMessages($topic, $pid, array($payload));
                $result = $this->producer->send();
                EdjLog::debugLog(__METHOD__.'|'.$topic.'|'.$pid);
            } catch (\Kafka\Exception $e) {
                EdjLog::error(__METHOD__.'|'.$e->getMessage());
                $this->getPartitionCount($topic, true);
            }
        }
    }
}

<?php
return array(
    'brokers' => "123.123.123.123:9092,123.123.123.123:9093,123.123.123.123:9094",  //ip一樣，埠不一樣
    //topic名的對映,推薦用class名字做key
    //測試環境和線上用不同的配置檔案
    'topicmap' => array(
        "RDriverPositionToKafka" => "driver_location_test",
        "ROrderToKafka" => "order_test",
        "SubmitOrderAutoService_saveOrderInfoJob" => "finished_order_picture",
        'vip_customer_change' => 'vip_customer_change',
     ),
);

連結：https://pan.baidu.com/s/1xiHAt8mbxpdPLhqZbKL1LQ 
提取碼：l92h  //kafka包

<?php
/**
 * 基於redis的queue佇列
 */
class Queue {
    private static $_models;

    public $queue_max_length = array(
    );

    public static function model($className=__CLASS__) {
        $model=null;
        if (isset(self::$_models[$className]))
            $model=self::$_models[$className];
        else {
            $model=self::$_models[$className]=new $className(null);
        }
        return $model;
    }

    //確定redis
    private function select_redis($type) {
        return QueuePool::model()->get_zone($type);
    }

    private function trim($queue_name) {

        $type = str_replace("queue_", "", $queue_name);
        $max = 0;
        if (isset($this->queue_max_length[$type])) {
            $max = intval($this->queue_max_length[$type]);
        }
        if ($max>0) {
            $zone = $this->select_redis($type);
            if($zone) {
                $zone['redis']->lTrim($queue_name, 0, $max-1);
            }
            else {
                EdjLog::error("can not find zone, queue name: " . $type);
                return;
            }
        }
    }

    /**
     * 放入佇列，統一佇列對外暴露方法,增加型別預設放task佇列，指定了就放對應的佇列，同時如果不在指定型別內的，也放預設佇列
     *
     * @author sunhongjing 2013-07-04
     * @param unknown_type $params
     * @param unknown_type $type
     * @return mixed
     */
    public function putin($params=null, $type){
        $type = empty($type) ? 'error' : strtolower($type);

                $base_qname = QNameManagerService::model()->get_base_qname($type);

        if(!empty($base_qname)) {
            $this->queue_name = 'queue_'.$base_qname;
        }else{
            $this->queue_name = 'queue_error';
        }

        if ($params===null) {
            return $this->get();
        } else {
            return $this->add($params);
        }
    }

    /**
     * 取一條佇列資料,封裝多個佇列，統一呼叫方法
     * @author sunhongjing 2013-07-09
     * @param string $type
     * @return array
     */
    public function getit($type='default')
    {
        $base_qname = QNameManagerService::model()->get_base_qname($type);

        if(!empty($base_qname)) {
            $this->queue_name = 'queue_'.$base_qname;
        }else{
            return array();
        }

        $zone = $this->select_redis($type);
        if($zone) {
            if($zone['brpop']) {
                $json = '';
                $result = $zone['redis']->brPop($this->queue_name, $zone['brpop']);
                if(!empty($result) && isset($result[1])) {
                    $json = $result[1];
                }
            }
            else {
                $json = $zone['redis']->rPop($this->queue_name);
            }
        }
        else {
            EdjLog::error("can not find zone, queue name: " . $type);
            return array();
        }

        return json_decode($json, true);
    }

    /**
     * 返回佇列接收的型別列表
     * @author sunhongjing 2013-07-04
     * @return array
     */
    public function getQueueTypeList()
    {
        $list = QNameManager::model()->findall();
        if($list) {
            return $list;
        }

        EdjLog::error("Error: get queue list from database");
        return array();
    }

    /**
     * 設定或者讀取位置佇列
     * @param array $params
     * @return mixed
     */
    public function position($params=null) {
        $this->queue_name='queue_position';

        if ($params===null) {
            return $this->get();
        } else {
            return $this->add($params);
        }
    }

    /**
     * 心跳佇列
     * @param string $params
     * @return mixed
     */
    public function heartbeat($params=null) {
        $this->queue_name='queue_heartbeat';

        if ($params===null) {
            return $this->get();
        } else {
            return $this->add($params);
        }
    }

    /**
     * 最高優先順序佇列
     * @param string $params
     * @return mixed
     */
    public function task($params=null) {
        $this->queue_name='queue_task';

        if ($params===null) {
            return $this->get();
        } else {
            return $this->add($params);
        }
    }

    /**
     * 儲存日誌到資料庫
     * @param string $params
     * @return mixed
     */
    public function dumplog($params=null) {
        $this->queue_name='queue_dumplog';

        if ($params===null) {
            return $this->get();
        } else {
            return $this->add($params);
        }
    }

    /**
     * 返回各個佇列中的任務總數
     */
    public function length() {

        $queue = $this->getQueueTypeList();

        $queue_length=array();
        $reg = "/P[0-9]+$/";
        foreach($queue as $item) {
            $base_qname = $item->base_qname;
            $zone = $this->select_redis($base_qname);
            $key = 'queue_'.$base_qname;
            if($zone) {
                $len = $zone['redis']->lLen($key);
                if(isset($item->max) && $len > $item->max) {
                    $key = '!'.$key;
                }

                $pkey = '';
                if(preg_match($reg, $zone['name'])) {
                    $pkey = $key.'@'.$zone['name'];
                }
                else {
                    $pkey = $key.'@'.$zone['name']."_P".$item->level;
                }

                $queue_length[$pkey] = $len;
            }
            else {
                EdjLog::error("can not find zone, queue name: " . $key);
            }
        }

        return $queue_length;
    }

    private function get() {
        $type = str_replace("queue_", "", $this->queue_name);
        $zone = $this->select_redis($type);
        if($zone) {
            if($zone['brpop']) {
                $json = '';
                $result = $zone['redis']->brPop($this->queue_name, $zone['brpop']);
                if(!empty($result) && isset($result[1])) {
                    $json = $result[1];
                }
            }
            else {
                $json = $zone['redis']->rPop($this->queue_name);
            }
        }
        else {
            EdjLog::error("can not find zone, queue name: " . $type);
            return array();
        }
        return json_decode($json, true);
    }

    private function add($params) {
        $json=json_encode($params);
        $type = str_replace("queue_", "", $this->queue_name);
        $zone = $this->select_redis($type);
        $return = 0;
        if($zone) {
            try {
                $return = $zone['redis']->lPush($this->queue_name, $json);
            } catch (Exception $e) {
                EdjLog::error("write redis error,msg:".$e->getMessage());
                //echo $e->getMessage();
            }
        }
        else {
            EdjLog::error("can not find zone, queue name: " . $type);
        }

        return $return;
    }

    public function processTask($task) {
        if(!isset($task['method'], $task['params'])) {
            $task_content = json_encode($task);
            EdjLog::error("can not run task due to no 'method' or 'params' specified, task is $task_content");
            return;
        }

        $method=$task['method'];
        $params=$task['params'];
        $class = isset($task['class']) ? $task['class'] : "QueueProcess";
        EdjLog::info("REDIS_QUEUE_OUT CLASS:$class METHOD:$method PARAMS:".json_encode($params));
        try {
            //throw new Exception("Value must be 1 or below");
            $queue_process=new $class();
            // check this method is exist, if not throw ReflectionException
            new ReflectionMethod($queue_process, $method);
            call_user_func_array(array($queue_process, $method), array($params));
        } catch(Exception $e) {
            $errmsg = $e->getMessage();
            EdjLog::error("execption queue_run method:$method err: $errmsg");
        }
    }

    public function getLengthByType($type){
        $type = empty($type) ? 'error' : strtolower($type);
        $base_qname = QNameManagerService::model()->get_base_qname($type);
        $zone = $this->select_redis($base_qname);
        $key = 'queue_'.$base_qname;
        $len = 0;
        if($zone) {
            $len = $zone['redis']->lLen($key);
        } else {
            EdjLog::error("can not find zone, queue name: " . $base_qname);
        }
        return $len;
    }
}

Kafka學習（四）-------- Kafka核心之Producer
2019-08-06
Kafka
Kafka學習（三）-------- Kafka核心之Cosumer
2019-07-25
Kafka
Kafka學習
2019-04-24
Kafka
Kafka 學習筆記（二）：初探 Kafka
2019-03-04
Kafka筆記
kafka學習（二）-------- 什麼是Kafka
2019-07-22
Kafka
Kafka學習筆記（二）：初探Kafka
2018-03-26
Kafka筆記
【kafka學習筆記】kafka的基本概念
2021-12-12
Kafka筆記
kafka基礎學習
2024-10-11
Kafka
Kafka 學習筆記
2022-02-07
Kafka筆記
Kafka 總結學習
2021-06-26
Kafka
Go操作kafka
2024-03-18
GoKafka
golang操作kafka
2024-07-01
GolangKafka
Kafka 學習筆記（一）：為什麼需要 Kafka？
2018-03-14
Kafka筆記
Kafka學習筆記（一）：為什麼需要Kafka？
2018-03-25
Kafka筆記
無鏡--kafka之消費者（四）
2021-09-09
Kafka
Kafka學習day01
2024-11-14
Kafka
Spring Kafka深入學習分析
2022-05-18
SpringKafka
kafka學習筆記（一）
2020-11-23
Kafka筆記
Kafka 入門（四）-- Python Kafka Client 效能測試
2021-01-16
KafkaPythonclient
從零開始學習Kafka
2019-05-10
Kafka
Go 操作kafka包sarama
2021-10-16
GoKafka
詳細解析kafka之kafka分割槽和副本
2021-09-09
Kafka
sparkStreaming 之 kafka源
2020-11-07
SparkKafka
kafka 測試遇到掉資料的問題 nmred/kafka-PHP
2019-12-17
KafkaPHP
訊息佇列Kafka學習總結
2018-07-11
佇列Kafka
PHP 使用 Kafka 安裝拾遺
2018-09-07
PHPKafka
php5.X連線kafka
2018-07-26
PHPKafka
PHP Kafka 訊息佇列使用
2020-05-12
PHPKafka佇列
Kafka之Producer原始碼
2018-08-14
Kafka原始碼
kafka原始碼剖析(二)之kafka-server的啟動
2018-03-15
Kafka原始碼Server
kafka-ngx_kafka_module
2020-11-16
Kafka
Kafka並不難學
2018-11-10
Kafka
kafka實戰教學
2018-03-26
Kafka
原始碼分析Kafka之Producer
2018-08-27
原始碼Kafka
Kafka之消費與心跳
2020-10-16
Kafka
Kafka原理剖析之「Topic建立」
2024-09-07
Kafka
docker compose of kafka and kafka-ui
2024-08-16
DockerKafkaUI
kafka之一：kafka簡介
2021-06-04
Kafka
訊息佇列之Kafka——從架構技術重新理解Kafka
2020-05-17
佇列Kafka架構

Kafka學習之（四）PHP操作Kafka

相關文章