MapReduce 實現 搜尋指數統計和找到人氣王

package com.mapreduce;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Partitioner;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;

/**

 *

 * 統計分別統計出男女明星最大搜尋指數

 */

public class Star extends Configured implements Tool {

/**

 *  Mapper 解析明星資料

 * @input key=偏移量  value=明星資料

 * @output key=gender value=name+hotIndex

 */

public static class ActorMapper extends Mapper<Object, Text, Text, Text> {

 

public void map(Object key, Text value, Context context) throws IOException, InterruptedException {

//value=name+gender+hotIndex

String[] tokens = value.toString().split("\t");//使用分隔符\t，將資料解析為陣列 tokens

String gender = tokens[1].trim();//性別

String nameHotIndex = tokens[0] + "\t" + tokens[2];//名稱和關注指數

//輸出key=gender value=name+hotIndex

context.write(new Text(gender), new Text(nameHotIndex));

}

}

 

/**

 *

 *  Partitioner 根據sex選擇分割槽

 *

 */

public static class ActorPartitioner extends Partitioner<Text, Text> {	 

        @Override

        public int getPartition(Text key, Text value, int numReduceTasks) {

        

            String sex = key.toString();//按性別分割槽

            

            // 預設指定分割槽 0

            if(numReduceTasks==0)

            	return 0;

            

            //性別為male 選擇分割槽0

            if(sex.equals("male"))             

                return 0;

            //性別為female 選擇分割槽1

            if(sex.equals("female"))

            	return 1 % numReduceTasks;

            //其他性別 選擇分割槽2

            else

                return 2 % numReduceTasks;

           

        }

    }

 

/**

 *

 *  定義Combiner 合併 Mapper 輸出結果

 *

 */

public static class ActorCombiner extends Reducer<Text, Text, Text, Text> {

private Text text = new Text();

@Override

public void reduce(Text key, Iterable<Text> values, Context context)throws IOException, InterruptedException {

int maxHotIndex = Integer.MIN_VALUE;

int hotIndex = 0;

String name="";

for (Text val : values) {

String[] valTokens = val.toString().split("\\t");

hotIndex = Integer.parseInt(valTokens[1]);

if(hotIndex>maxHotIndex){

name = valTokens[0];

maxHotIndex = hotIndex;

}

}

text.set(name+"\t"+maxHotIndex);

context.write(key, text);

}

}

/**

 *

 *  Reducer 統計男、女明星最高搜尋指數

 * @input key=gender  value=name+hotIndex

 * @output key=name value=gender+hotIndex(max)

 */

public static class ActorReducer extends Reducer<Text, Text, Text, Text> {

 

@Override

public void reduce(Text key, Iterable<Text> values, Context context)throws IOException, InterruptedException {

 

int maxHotIndex = Integer.MIN_VALUE;

 

String name = " ";

int hotIndex = 0;

// 根據key，迭代 values 集合，求出最高搜尋指數

for (Text val : values) {

String[] valTokens = val.toString().split("\\t");

hotIndex = Integer.parseInt(valTokens[1]);

if (hotIndex > maxHotIndex) {

name = valTokens[0];

maxHotIndex = hotIndex;

}

}

context.write(new Text(name), new Text( key + "\t"+ maxHotIndex));

}

}

 

/**

 *  任務驅動方法

 * @param args

 * @return

 * @throws Exception

 */

@Override

public int run(String[] args) throws Exception {

// TODO Auto-generated method stub

Configuration conf = new Configuration();//讀取配置檔案

Path mypath = new Path(args[1]);

FileSystem hdfs = mypath.getFileSystem(conf);

if (hdfs.isDirectory(mypath)) {

hdfs.delete(mypath, true);

}

 

Job job = new Job(conf, "star");//新建一個任務

job.setJarByClass(Star.class);//主類

job.setNumReduceTasks(2);//reduce的個數設定為2

job.setPartitionerClass(ActorPartitioner.class);//設定Partitioner類

job.setMapperClass(ActorMapper.class);//Mapper

job.setMapOutputKeyClass(Text.class);//map 輸出key型別

job.setMapOutputValueClass(Text.class);//map 輸出value型別

job.setCombinerClass(ActorCombiner.class);//設定Combiner類

job.setReducerClass(ActorReducer.class);//Reducer

job.setOutputKeyClass(Text.class);//輸出結果 key型別

job.setOutputValueClass(Text.class);//輸出結果 value型別

FileInputFormat.addInputPath(job, new Path(args[0]));// 輸入路徑

FileOutputFormat.setOutputPath(job, new Path(args[1]));// 輸出路徑

job.waitForCompletion(true);//提交任務

return 0;

}

/**

 *  main 方法

 * @param args

 * @throws Exception

 */

public static void main(String[] args) throws Exception {

String[] args0 = { "hdfs://cdh001:9000/actor/actor.txt",

"hdfs://cdh001:9000/actor/out/" };

int ec = ToolRunner.run(new Configuration(), new Star(), args0);

System.exit(ec);

}

}