Hbase篇--Hbase和MapReduce結合Api

LHBlog發表於2018-01-16
API

一.前述

Mapreduce可以自定義Inputforma物件和OutPutformat物件，所以原理上Mapreduce可以和任意輸入源結合。

二.步驟

將結果寫會到hbase中去。

 2.1 Main函式

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;


/**
 * 分析hdfs 文字  統計單詞數量
 * 結果輸出到 hbase表
 * create 'wc','cf'
 * rowkey: 單詞        cf:count=單詞數量
 * @author root
 *
 */
public class WCDemo {

    /**
     * 
     * wc
     * 資料hbase表    rowkey  cell存放文字
     * 結果輸出到 hbase表
     * 
     */

    public static void main(String[] args) throws Exception {
        
        Configuration conf = new Configuration();
        
        conf.set("fs.defaultFS", "hdfs://node1:8020");//設定hdfs叢集nameservices名稱
        conf.set("hbase.zookeeper.quorum", "node4");
        
        Job job = Job.getInstance(conf);
        
        job.setJarByClass(WCDemo.class);
        
        job.setMapperClass(WCMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        
//        job.setReducerClass();
        
        //addDependencyJars  本地方式執行： 設定為false
//        TableMapReduceUtil.initTableReducerJob("wc", WCReducer.class, job);
        TableMapReduceUtil.initTableReducerJob("wc",WCReducer.class, job,
                null, null, null, null, false);
        
        Path path = new Path("/user/wc");
        FileInputFormat.addInputPath(job, path);
        
        boolean flag = job.waitForCompletion(true);
        if(flag) {
            System.out.println("success~~");
        }
    }
    
}

2.2 Mapper函式（和正常的Mapper沒啥區別）

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class WCMapper extends Mapper<LongWritable, Text, Text, IntWritable> {

    @Override
    protected void map(LongWritable key, Text value, Context context)
            throws IOException, InterruptedException {
        String[] words = value.toString().split(" ");
        
        for (String w : words) {
            context.write(new Text(w), new IntWritable(1));
        }
    }
}

2.3 Reduce函式（主要是把Put物件寫出去）

import java.io.IOException;

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;

public class WCReducer extends
        TableReducer<Text, IntWritable, ImmutableBytesWritable> {

    @Override
    protected void reduce(Text text, Iterable<IntWritable> iterable,
            Context context) throws IOException, InterruptedException {
        
        int sum = 0;
        
        for (IntWritable i : iterable) {
            sum += i.get();
        }
        
        Put put = new Put(text.toString().getBytes());
        put.add("cf".getBytes(), "count".getBytes(), (sum+"").getBytes());
        
        context.write(null, put);
    }
}

 

相關文章