DATAX

DataX
DataX 是阿里巴巴集團內被廣泛使用的離線資料同步工具/平臺，實現包括 MySQL、Oracle、SqlServer、Postgre、HDFS、Hive、ADS、HBase、TableStore(OTS)、MaxCompute(ODPS)、DRDS 等各種異構資料來源之間高效的資料同步功能。

下載安裝datax.tar，下載連結

MysqlWriter 外掛實現了寫入資料到 Mysql 主庫的目的表的功能。在底層實現上， MysqlWriter 透過 JDBC 連線遠端 Mysql 資料庫，並執行相應的 insert into ... 或者 ( replace into ...) 的 sql 語句將資料寫入 Mysql，內部會分批次提交入庫，需要資料庫本身採用 innodb 引擎。

MysqlWriter 面向ETL開發工程師，他們使用 MysqlWriter 從數倉匯入資料到 Mysql。同時 MysqlWriter 亦可以作為資料遷移工具為DBA等使用者提供服務。

[root@scripts]# tar -xvf datax.tar

[root@datax]# ls
bin conf job lib log log_perf plugin script tmp

[root@job]# ls
job.json

[root@job]# vi mysql2mysql.json

[root@job]# cat mysql2mysql.json 
{
"job": {
"setting": {
"speed": {
"channel": 8
},
"errorLimit": {
"record": 0,
"percentage": 0.02
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "username",
"password": "password",
"column": [
"order_id",
"parent_order_id",
"chartered_id",
"multi_order_id",
"carpool_order_id",
"order_no",
"city_id",
"type",
"order_service_type",
"booking_user_id",
"booking_phone",
"rider_phone",
"booking_start_time",
"booking_start_addr",
"booking_end_addr",
"booking_start_short_addr",
"booking_end_short_addr",
"driver_id",
"license_plates",
"order_status",
"channels_num",
"is_display",
"dissent_flag",
"is_carpool",
"create_date",
"update_date",
"sub_order_count",
"sub_order_flag",
"push_driver_type",
"buyout_flag",
"is_order_others",
"auto_level_up",
"finished_date",
"order_create_date",
"order_update_date",
"cancel_type",
"business_id",
"order_type",
"fact_date",
"fact_start_addr",
"fact_end_addr",
"cancelorder_penalty"
],
"connection": [
{
"table": [
"yoon_table" #表名
],
"jdbcUrl": [
"jdbc:mysql://192.168.11.111:3306/yoon" #連線地址、埠、資料庫
]
}
],
"where": "order_create_date >='2019-05-01 00:00:00' and order_create_date < '2019-06-01 00:00:00'" #根據where條件傳輸資料，全表導的話就把where條件去掉即可
}
},
"writer": {
"name": "mysqlwriter",
"parameter": {
"writeMode": "replace",
"username": "username",
"password": "password",
"column": [
"order_id",
"parent_order_id",
"chartered_id",
"multi_order_id",
"carpool_order_id",
"order_no",
"city_id",
"type",
"order_service_type",
"booking_user_id",
"booking_phone",
"rider_phone",
"booking_start_time",
"booking_start_addr",
"booking_end_addr",
"booking_start_short_addr",
"booking_end_short_addr",
"driver_id",
"license_plates",
"order_status",
"channels_num",
"is_display",
"dissent_flag",
"is_carpool",
"create_date",
"update_date",
"sub_order_count",
"sub_order_flag",
"push_driver_type",
"buyout_flag",
"is_order_others",
"auto_level_up",
"finished_date",
"order_create_date",
"order_update_date",
"cancel_type",
"business_id",
"order_type",
"fact_date",
"fact_start_addr",
"fact_end_addr",
"cancelorder_penalty"
],
"connection": [
{
"jdbcUrl": "jdbc:mysql://192.168.11.222:3306/hank?useUnicode=true&characterEncoding=utf8", #連線地址、埠、資料庫
"table": [
"yoon_hank" #表名
]
}
]
}
}
}
]
}
}