細節決定成敗 MapReduce任務實戰 Map Join

self_control發表於2016-06-14

一、任務描述

在hdfs 上有兩個檔案

學生資訊檔案：hdfs://***.***.***:8020/user/train/joinjob/student.txt

以逗號分隔，第一列為學號，學號每個學生唯一，第二列為學生姓名。

點選(此處)摺疊或開啟

  1. 2016001,Join
  2. 2016002,Abigail
  3. 2016003,Abby
  4. 2016004,Alexandra
  5. 2016005,Cathy
  6. 2016006,Katherine

學生成績資訊檔案：hdfs://***.***.***:8020/user/train/joinjob/student_score.txt

以逗號分隔，第一列為學號，學號每個學生唯一，第二列為課程程式碼，第三列為分數。

點選(此處)摺疊或開啟

  1. 2016001,YY,60
  2. 2016001,SX,88
  3. 2016001,YW,91
  4. 2016002,SX,77
  5. 2016002,YW,33
  6. .............

期望的結果檔案形式如下

點選(此處)摺疊或開啟

  1. 2016001,Join,YY,60
  2. 2016001,Join,SX,88
  3. 2016001,Join,YW,91
  4. 2016002,Abigail,SX,77
  5. 2016002,Abigail,YW,33
  6. 2016002,Abigail,YY,56
  7. 2016003,Abby,YY,34
  8. 2016003,Abby,SX,84
  9. 2016003,Abby,YW,69
  10. 2016004,Alexandra,YY,89
  11. 2016004,Alexandra,SX,84
  12. .......

二、任務分析

  1. 這是一個兩個資料集關聯的任務。關聯有map端join ，和reduce端join。 通常在有一個資料集比較小，可在全部放到記憶體中時，選擇map端join；當兩個資料集都很大時，使用reduce端join，這時可以使用Bloom過濾器來增加效率。 本次任務我們使用map端join實現。
  2. 此任務關聯到直接輸出即可，不需要reduce過程。
  3. 本任務會使用到的技術有：分散式快取DistributedCache、mapreduce中的本地檔案讀取

三、實現思路

  1. 把一個學生資訊使用分散式快取分發到各個任務節點。
  2. 學生成績資訊當做map的input。
  3. 在mapper 的setup函式中讀取已快取到本的學生資訊，並儲存到一個HashMap中，key為學號，value為姓名。
  4. 在mapper函式中對每一個學生成績資訊，根據學號去上一步的map中去取姓名資訊。
  5. 封裝到Text物件中輸出。

三、實現程式碼

點選(此處)摺疊或開啟

  1. package join;
  2. import java.io.BufferedReader;
  3. import java.io.IOException;
  4. import java.io.InputStreamReader;
  5. import java.net.URI;
  6. import java.util.HashMap;
  7. import java.util.Map;

  9. import org.apache.commons.lang.StringUtils;
  10. import org.apache.hadoop.conf.Configuration;
  11. import org.apache.hadoop.conf.Configured;
  12. import org.apache.hadoop.fs.FileSystem;
  13. import org.apache.hadoop.fs.Path;
  14. import org.apache.hadoop.io.LongWritable;
  15. import org.apache.hadoop.io.NullWritable;
  16. import org.apache.hadoop.io.Text;
  17. import org.apache.hadoop.mapreduce.Job;
  18. import org.apache.hadoop.mapreduce.Mapper;
  19. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  20. import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
  21. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  22. import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
  23. import org.apache.hadoop.util.Tool;
  24. import org.apache.hadoop.util.ToolRunner;
  25. import org.apache.log4j.Logger;


  28. public class MapJoinDirectOutPutJob extends Configured implements Tool {
  29.         private static String STUDENTNAME_PARAM ="STUDENTNAME_PARAM";
  30.         private static String INPATH_SCORE = "joinjob/student_score.txt";
  31.         private static String OUTPATH = "joinjob/output";
  32.         public static class MapJoinDirectOutPutMapper extends Mapper<LongWritable,Text,NullWritable,Text>{
  33.                 private BufferedReader br = null;
  34.                 private Map <String,String> map =new HashMap<String,String>();
  35.                 private Text newValue = new Text();
  36.                 @Override
  37.                 protected void map(LongWritable key, Text value, Context context)
  38.                                 throws IOException, InterruptedException {
  39.                         String [] words = StringUtils.split(value.toString(),',');
  40.                         String name = map.get(words[0]);
  41.                         newValue.set(words[0]+","+name +","+words[1]+","+words[2]);
  42.                         context.write(NullWritable.get(), newValue);
  43.                 }

  45.                 @Override
  46.                 protected void setup(Context context) throws IOException,
  47.                                 InterruptedException {
  48.                         Configuration conf = context.getConfiguration();
  49.                         FileSystem fs = FileSystem.getLocal(conf);

  51.                         br = new BufferedReader(new InputStreamReader(fs.open(new Path("studentLink.txt"))));
  52.                         String current;
  53.                         while((current= br.readLine())!=null){
  54.                                 String [] words = current.split(",");
  55.                                 map.put(words[0],words[1]);
  56.                         }
  57.                         br.close();
  58.                 }

  60.         }

  62.         @Override
  63.         public int run(String[] args) throws Exception {
  64.                 Job job = Job.getInstance(getConf(),"MapJoinDirectOutPutJob");
  65.                 job.setJarByClass(getClass());
  66.                 Configuration conf = job.getConfiguration();

  68.                 conf.set(STUDENTNAME_PARAM, args[0]);

  70.                 Path in = new Path(INPATH_SCORE);
  71.                 Path out = new Path(OUTPATH);
  72.                 FileSystem.get(conf).delete(out,true);
  73.                 FileInputFormat.setInputPaths(job, in);
  74.                 FileOutputFormat.setOutputPath(job, out);
  75.                 job.setInputFormatClass(TextInputFormat.class);
  76.                 job.setOutputFormatClass(TextOutputFormat.class);
  77.                 job.setMapperClass(MapJoinDirectOutPutMapper.class);
  78.                 job.setMapOutputKeyClass(NullWritable.class);
  79.                 job.setMapOutputValueClass(Text.class);

  81.                 job.setNumReduceTasks(0);
  82.                 URI uri = new URI("hdfs://***.***.***.***:8020/user/train/joinjob/student.txt#studentLink.txt");
  83.                 job.addCacheFile(uri);

  85.                 return job.waitForCompletion(true) ?0:1;
  86.         }

  88.         public static void main(String [] args){
  89.                 int r = 0;
  90.                 try{
  91.                         r = ToolRunner.run(new Configuration(),new MapJoinDirectOutPutJob(),args);
  92.                 }catch(Exception e){
  93.                         e.printStackTrace();
  94.                 }
  95.                 System.exit(r);
  96.         }

  98. }
本文地址：http://blog.itpub.net/30066956/viewspace-2120063/



來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/30066956/viewspace-2120063/，如需轉載，請註明出處，否則將追究法律責任。

相關文章