Flink CDC實戰

1，關於Flink cdc的使用說明

  1）匯入依賴

<dependency>
  <groupId>com.alibaba.ververica</groupId>
  <artifactId>flink-connector-mysql-cdc</artifactId>
  <version>1.1.0</version>
</dependency>

  sql案例 ：

 

  2）需要理解注意的地方

    

 問題：使用cdc的時候會影響mysql的效能跟正常使用嗎？

      不會，這個鎖應該很快就能釋放，沒什麼效能瓶頸，但是得保證你接binlog的使用者開了reload許可權

  請百度 ‘’mysql的reload許可權‘

2，官網的案例程式碼

import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import com.alibaba.ververica.cdc.connectors.mysql.MySQLSource;


public class MySqlBinlogSourceExample {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
            SourceFunction<String> sourceFunction = MySQLSource.<String>builder()
                    .hostname("192.168.5.24")
                    .port(3306)
                    .databaseList("cdc") // monitor all tables under inventory database

                    .username("root")
                    .password("Sjb@123456")
                    .deserializer(new CdcDeserializationSchema()) // converts SourceRecord to String
                    .build();


        DataStreamSource<String> stringDataStreamSource = env.addSource(sourceFunction);

        stringDataStreamSource.print("===>");


        try {
            env.execute("測試mysql-cdc");
        } catch (Exception e) {
            e.printStackTrace();
        }

    }
}

3，紅色標註的地方我們自定義一個類，解析一下cdc的格式，然後拼接成自己需要的格式

import com.alibaba.ververica.cdc.debezium.DebeziumDeserializationSchema;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.util.Collector;
import org.apache.kafka.connect.data.Field;
import org.apache.kafka.connect.data.Schema;
import org.apache.kafka.connect.data.Struct;
import org.apache.kafka.connect.source.SourceRecord;

import java.util.List;

/**
 * @program: flink-neiwang-dev
 * @description:
 * @author: Mr.Wang
 * @create: 2020-10-21 16:04
 **/
public class CdcDeserializationSchema implements DebeziumDeserializationSchema<String> {
    private static final long serialVersionUID = -3168848963265670603L;

    public CdcDeserializationSchema() {
    }

    @Override
    public void deserialize(SourceRecord record, Collector<String> out) throws Exception {
        Struct dataRecord  =  (Struct)record.value();
        Struct afterStruct = dataRecord.getStruct("after");
        List<Field> fieldsList = afterStruct.schema().fields();

        //todo 欄位與值
        for (Field field : fieldsList) {
            String fieldName = field.name();
            Object fieldValue = afterStruct.get(fieldName);
            System.out.println("*****fieldName=" + fieldName+",fieldValue="+fieldValue);
        }
        System.out.println("value = " + dataRecord);
        Schema schema = record.keySchema();

        //todo 拿到topic
        String topic = record.topic();
        System.out.println("topic = " + topic);

        //todo 主鍵欄位
        Struct pk = (Struct)record.key();
        List<Field> pkFieldList = pk.schema().fields();
        for (Field field : pkFieldList) {
            Object pkValue= pk.get(field.name());
            System.out.println("&&&&pkKey=" + field.name()+",pkValue="+pkValue);
        }

        //todo 需要輸出的值，這裡自己定義格式吧
        
        out.collect(record.toString());
    }
    @Override
    public TypeInformation<String> getProducedType() {
        return BasicTypeInfo.STRING_TYPE_INFO;
    }
}

 

 

參考：

https://github.com/ververica/flink-cdc-connectors/wiki/MySQL-CDC-Connector

https://mp.weixin.qq.com/s/Mfn-fFegb5wzI8BIHhNGvQ