MapReduce 二次排序詳解

package sorttwice;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

public class sorttwice {

	//自己定義的key類實現WritableComparable介面
	public static class IntPair implements WritableComparable{

		int first,second;
		/**
		 * 設定 left,right
		 */
		public void set(int left,int right){
			first=left;
			second=right;
		}
		public int getFirst(){
			return first;
		}
		public int getSecond(){
			return second;
		}
		//反序列化，從流中讀進二進位制轉換成IntPair
		@Override
		public void readFields(DataInput in) throws IOException {
			// TODO Auto-generated method stub
			first = in.readInt();
			second = in.readInt();
		}
		//序列化，將IntPair轉換成二進位制輸出
		@Override
		public void write(DataOutput out) throws IOException {
			// TODO Auto-generated method stub
			out.writeInt(first);
			out.writeInt(second);
		}
		//key的比較
		@Override
		public int compareTo(IntPair o) {
			// TODO Auto-generated method stub
			if(first!=o.first){
				return first{

		@Override
		public int getPartition(IntPair key, IntWritable value, int numParatitions) {
			// TODO Auto-generated method stub
			return Math.abs(key.getFirst()*127) % numParatitions;
		}
	}
	/**
     * 分組函式類。只要first相同就屬於同一個組。
     */
    //第一種方法，實現介面RawComparator
    public static class GroupingComparator implements RawComparator {
        @Override
        public int compare(IntPair o1, IntPair o2) {
            int l = o1.getFirst();
            int r = o2.getFirst();
            return l == r ? 0 : (l < r ? -1 : 1);
        }
        @Override
        //一個位元組一個位元組的比，直到找到一個不相同的位元組，然後比這個位元組的大小作為兩個位元組流的大小比較結果。
        public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){
            // TODO Auto-generated method stub
             return WritableComparator.compareBytes(b1, s1, Integer.SIZE/8,
                     b2, s2, Integer.SIZE/8);
        }
    }

	/*
	//第二種方法：繼承WritableComparator
	public static class GroupingComparator extends WritableComparator{
		private GroupingComparator(){
			super(IntPair.class,true);
		}
		@Override
		//Compare two WritableComparatles
		public int compare(WritableComparable w1,WritableComparable w2){
			IntPair ip1 = (IntPair) w1;
			IntPair ip2 = (IntPair) w2;
			int l = ip1.getFirst();
			int r = ip2.getSecond();
			return l == r ? 0 : (l{
		private final IntPair intkey = new IntPair();
		private final IntWritable intvalue = new IntWritable();
		public void map(Object key,Text value,Context context) throws IOException, InterruptedException{
			StringTokenizer itr = new StringTokenizer(value.toString());
			int left =0;
			int right =0;
			while(itr.hasMoreTokens()){
				left = Integer.parseInt(itr.nextToken());
				if(itr.hasMoreTokens())
					right = Integer.parseInt(itr.nextToken());
				intkey.set(left, right);
				intvalue.set(right);
				context.write(intkey, intvalue);
				
			}
		}
	}
	
	public static class Reduce extends Reducer{
		private final Text left = new Text();
		private static final Text SEPAPATOR= new Text("================================");
		public void reduce(IntPair key,Iterablevalues,Context context) throws IOException, InterruptedException{
			context.write(SEPAPATOR, null);
			left.set(Integer.toString(key.getFirst()));
			for(IntWritable val:values){
				context.write(left, val);
			}
		}
	}
	
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		// TODO Auto-generated method stub

		//讀取hadoop配置
		Configuration conf = new Configuration();
		
		//初始化作業
		Job job =new Job(conf,"Secondray Sort");
		job.setJarByClass(sorttwice.class);
		
		job.setNumReduceTasks(1);
		
		//設定map和reduce類
		job.setMapperClass(Map.class);
		job.setReducerClass(Reduce.class);
		
		//設定map的輸出key類,value類
		job.setMapOutputKeyClass(IntPair.class);
		job.setMapOutputValueClass(IntWritable.class);
		
		//設定輸出Reduce的key，value
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		
		//將輸入的資料分割成小數塊，並使用RecordReader類實現
		job.setInputFormatClass(TextInputFormat.class);
		
		//提供一個RecordReader實現資料的輸出
		job.setOutputFormatClass(TextOutputFormat.class);
		
		//重寫分割槽函式
		job.setPartitionerClass(FirstPartition.class);
		
		//重寫分組函式
		job.setGroupingComparatorClass(GroupingComparator.class);
		
		//設定路徑
		FileInputFormat.addInputPath(job, new Path("/thinkgamer/input"));
		FileOutputFormat.setOutputPath(job,new Path("/thinkgamer/output"));
		
		//提交作業
		System.exit(job.waitForCompletion(true)?0:1);
	}
}