一、flume 日誌採集

1. flume分散式安裝（略）

本文采用 apache-flume-1.9.0，一主兩從搭建，通過負載均衡方式將採集到的日誌輸入hbase儲存

2. 基本配置

(1) # master 通過exec 方式監聽日誌檔案輸出

[root@master conf]# cd /usr/local/apache-flume-1.9.0-bin
[root@master conf]# vim conf/flume-client.conf

# agent1 name
agent1.channels = c1
agent1.sources = r1
agent1.sinks = k1 k2

# set channel
agent1.channels.c1.type = memory
agent1.channels.c1.capacity = 1000
agent1.channels.c1.transactionCapacity = 100

# set source # exec監控command
agent1.sources.r1.channels = c1
agent1.sources.r1.type = exec
agent1.sources.r1.command = tail -F /usr/local/apache-flume-1.9.0-bin/data/test_cluster.log

# set sink1
agent1.sinks.k1.channel = c1
agent1.sinks.k1.type = avro
agent1.sinks.k1.hostname = slave1
agent1.sinks.k1.port = 52020

# set sink2
agent1.sinks.k2.channel = c1
agent1.sinks.k2.type = avro
agent1.sinks.k2.hostname = slave2
agent1.sinks.k2.port = 52020

# set sink group
agent1.sinkgroups = g1
agent1.sinkgroups.g1.sinks = k1 k2

# load balance
agent1.sinkgroups.g1.processor.type = load_balance
agent1.sinkgroups.g1.processor.selector = round_robin

# set failover
#agent1.sinkgroups.g1.processor.type = failover
#agent1.sinkgroups.g1.processor.k1 = 10
#agent1.sinkgroups.g1.processor.k2 = 1
#agent1.sinkgroups.g1.processor.maxpenalty = 10000

# 啟動master flume 節點

[root@master apache-flume-1.9.0-bin]# bin/flume-ng agent --conf conf --conf-file conf/flume-client.conf --name agent1 -Dflume.root.logger=INFO,console

(2) #slave1 節點配置

[root@slave1 home]# cd /usr/local/apache-flume-1.9.0-bin
[root@slave1 apache-flume-1.9.0-bin]# vim conf/flume-hbase.conf

# agent1 name
a1.channels = c1
a1.sources = r1
a1.sinks = k1

# set channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# other node, slave to master
a1.sources.r1.type = avro
a1.sources.r1.bind = slave1
a1.sources.r1.port = 52020

# set sink to hdfs
a1.sinks.k1.type = org.apache.flume.sink.hbase.HBaseSink
a1.sinks.k1.table = t_user
a1.sinks.k1.columnFamily = user_profile
a1.sinks.k1.serializer = org.apache.flume.sink.hbase.RegexHbaseEventSerializer
a1.sinks.k1.serializer.regex = \\[(.*?)\\]\\ \\[(.*?)\\]\\ \\[(.*?)\\]\\ \\[(.*?)\\]\\ \\[(.*?)\\]
a1.sinks.k1.serializer.colNames = userId,gender,province,birthday,lastLoginTime

a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

# 啟動slave1 flume 節點

[root@slave1 apache-flume-1.9.0-bin]# bin/flume-ng agent --conf conf --conf-file conf/flume-server.conf --name a1 -Dflume.root.logger=INFO,console

(3) #slave2 節點配置

[root@slave2 home]# cd /usr/local/apache-flume-1.9.0-bin
[root@slave2 apache-flume-1.9.0-bin]# vim conf/flume-hbase.conf

# agent1 name
a1.channels = c1
a1.sources = r1
a1.sinks = k1

# set channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# other node, slave to master
a1.sources.r1.type = avro
a1.sources.r1.bind = slave2
a1.sources.r1.port = 52020

# set sink to hdfs
a1.sinks.k1.type = org.apache.flume.sink.hbase.HBaseSink
a1.sinks.k1.table = t_user
a1.sinks.k1.columnFamily = user_profile
a1.sinks.k1.serializer = org.apache.flume.sink.hbase.RegexHbaseEventSerializer
a1.sinks.k1.serializer.regex = \\[(.*?)\\]\\ \\[(.*?)\\]\\ \\[(.*?)\\]\\ \\[(.*?)\\]\\ \\[(.*?)\\]
a1.sinks.k1.serializer.colNames = userId,gender,province,birthday,lastLoginTime

a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

# 啟動slave2 flume 節點

[root@slave2 apache-flume-1.9.0-bin]# bin/flume-ng agent --conf conf --conf-file conf/flume-server.conf --name a1 -Dflume.root.logger=INFO,console

二、hbase 資料儲存

1. hbase 叢集安裝部署(略)

2. shell終端啟動hbase,並建立表 t_user

3. 向表 t_user手動插入一些資料

4. 檢視錶 t_user資料

三、hive 查詢輸出

1. hive 叢集安裝部署(略)

2. 終端啟動hive

3. 建立hive到hbase的對映表

create external table t_user(id string, userId string, gender string, province string, birthday string, lastLoginTime string)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES (
"hbase.columns.mapping" = ":key,user_profile:userId,user_profile:gender,user_profile:province,user_profile:birthday,user_profile:lastLoginTime")
TBLPROPERTIES("hbase.table.name" = "t_user");