一個MapReduce 程式示例 細節決定成敗(七) ：自定義Key 及RecordReader

self_control發表於2016-05-30
上一篇中，演示瞭如何使用CombineFileInputFormat 來優化當有多個輸入小檔案時，減少起動的map task個數。
在自定義的MyCombineFileInputFormat中的MyRecordReader是簡單代理了LineRecordReader。
其它我也還可以在這個地方做更多的東西。
本次實驗是使用自定義的RecordReader從split中自定義 key value。

自定義MyKey
自定義的key 需要實現WritableComparable  介面。

點選(此處)摺疊或開啟

  1. package wordcount;

  3. import java.io.DataInput;
  4. import java.io.DataOutput;
  5. import java.io.IOException;

  7. import org.apache.hadoop.io.WritableComparable;

  9. public class MyKey implements WritableComparable<MyKey> {

  11.         private char c;
  12.         @Override
  13.         public void write(DataOutput out) throws IOException {
  14.                 out.writeChar(c);
  15.         }

  17.         @Override
  18.         public void readFields(DataInput in) throws IOException {
  19.                 c= in.readChar();
  20.         }

  22.         @Override
  23.         public int compareTo(MyKey key) {
  24.                 if(c==key.c)
  25.                         return 0;
  26.                 else if(c> key.c)
  27.                         return 1;
  28.                 else
  29.                         return -1;
  30.         }

  32.         public char getC() {
  33.                 return c;
  34.         }

  36.         public void setC(char c) {
  37.                 this.c = c;
  38.         }


  41. }

自定義CombinedFilesInputFormat 自定義RecordReader

點選(此處)摺疊或開啟

  1. package wordcount;

  3. import java.io.IOException;

  5. import org.apache.commons.lang.StringUtils;
  6. import org.apache.hadoop.io.IntWritable;
  7. import org.apache.hadoop.io.LongWritable;
  8. import org.apache.hadoop.io.Text;
  9. import org.apache.hadoop.mapreduce.InputSplit;
  10. import org.apache.hadoop.mapreduce.RecordReader;
  11. import org.apache.hadoop.mapreduce.TaskAttemptContext;
  12. import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat;
  13. import org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader;
  14. import org.apache.hadoop.mapreduce.lib.input.CombineFileSplit;
  15. import org.apache.hadoop.mapreduce.lib.input.FileSplit;
  16. import org.apache.hadoop.mapreduce.lib.input.LineRecordReader;
  17. import org.apache.log4j.Logger;


  20. public class MyCombinedFilesInputFormat extends CombineFileInputFormat<LongWritable, Text> {

  22.         @SuppressWarnings({ "unchecked", "rawtypes" })
  23.         @Override
  24.         public RecordReader<LongWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException {
  25.                 return new CombineFileRecordReader((CombineFileSplit) split,context,MyCombinedFilesRecordReader.class);
  26.         }

  28.         public static class MyCombinedFilesRecordReader extends RecordReader<MyKey, IntWritable> {
  29.                 private int index;
  30.                 private LineRecordReader reader;

  32.                 private String tValue;
  33.                 private int pos=0;

  35.                 private MyKey key = new MyKey();

  37.                 Logger log = Logger.getLogger(MyCombinedFilesRecordReader.class);
  38.                 public MyCombinedFilesRecordReader(CombineFileSplit split, TaskAttemptContext context, Integer index) {
  39.                         this.index = index;
  40.                         reader = new LineRecordReader();
  41.                 }

  43.                 @Override
  44.                 public void initialize(InputSplit split, TaskAttemptContext context)
  45.                                 throws IOException, InterruptedException {
  46.                         CombineFileSplit cfsplit = (CombineFileSplit) split;
  47.                         FileSplit fileSplit = new FileSplit(cfsplit.getPath(index),
  48.                                                                                                 cfsplit.getOffset(index),
  49.                                                                                                 cfsplit.getLength(index),
  50.                                                                                                 cfsplit.getLocations()
  51.                                         );
  52.                         reader.initialize(fileSplit, context);
  53.                 }

  55.                 @Override
  56.                 public boolean nextKeyValue() throws IOException, InterruptedException {
  57.                         if(StringUtils.isEmpty(tValue)||pos>=tValue.length()-1){
  58.                                 if(reader.nextKeyValue()){
  59.                                         pos = 0;
  60.                                         this.tValue = reader.getCurrentValue().toString();
  61.                                         return true;
  62.                                 }
  63.                                 else{
  64.                                         return false;
  65.                                 }
  66.                         }
  67.                         else{
  68.                                 pos ++;
  69.                                 if(tValue.charAt(pos)<='z' && tValue.charAt(pos)>='A'){
  70.                                         return true;
  71.                                 }
  72.                                 else{
  73.                                         return nextKeyValue();
  74.                                 }
  75.                         }
  76.                 }

  78.                 @Override
  79.                 public MyKey getCurrentKey() throws IOException,
  80.                                 InterruptedException {
  81.                         key.setC(tValue.charAt(pos));
  82.                         return key;
  83.                 }

  85.                 @Override
  86.                 public IntWritable getCurrentValue() throws IOException, InterruptedException {
  87.                         return new IntWritable(1);
  88.                 }

  90.                 @Override
  91.                 public float getProgress() throws IOException, InterruptedException {
  92.                         return reader.getProgress();
  93.                 }

  95.                 @Override
  96.                 public void close() throws IOException {
  97.                         reader.close();
  98.                 }

  100.         }
  101. }
Mapper程式程式碼
可以看到使用MyRecordReader返回自定義Key後，Map 函式得到了很大的簡化。

點選(此處)摺疊或開啟

  1. public static class MyWordCountMapper extends
  2.                         Mapper<MyKey, NullWritable, Text, IntWritable> {
  3.                 Text mKey = new Text();
  4.                 IntWritable mValue = new IntWritable(1);
  5.                 @Override
  6.                 protected void map(MyKey key, NullWritable value, Context context)
  7.                                 throws IOException, InterruptedException {
  8.                         mKey.set(String.valueOf(key.getC()));
  9.                         context.write(mKey, mValue);
  10.                 }
  11.         }
為了方便以後檢視，把主程式程式碼也貼上來。

點選(此處)摺疊或開啟

  1. package wordcount;

  3. import java.io.IOException;

  5. import org.apache.hadoop.conf.Configuration;
  6. import org.apache.hadoop.conf.Configured;
  7. import org.apache.hadoop.fs.Path;
  8. import org.apache.hadoop.io.IntWritable;
  9. import org.apache.hadoop.io.NullWritable;
  10. import org.apache.hadoop.io.Text;
  11. import org.apache.hadoop.mapreduce.Job;
  12. import org.apache.hadoop.mapreduce.Mapper;
  13. import org.apache.hadoop.mapreduce.Reducer;
  14. import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
  15. import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
  16. import org.apache.hadoop.util.Tool;
  17. import org.apache.hadoop.util.ToolRunner;
  18. import org.apache.log4j.Logger;

  20. public class MyWordCountJob extends Configured implements Tool {
  21.         Logger log = Logger.getLogger(MyWordCountJob.class);

  23.         public static class MyWordCountMapper extends
  24.                         Mapper<MyKey, NullWritable, Text, IntWritable> {
  25.                 Text mKey = new Text();
  26.                 IntWritable mValue = new IntWritable(1);
  27.                 @Override
  28.                 protected void map(MyKey key, NullWritable value, Context context)
  29.                                 throws IOException, InterruptedException {
  30.                         mKey.set(String.valueOf(key.getC()));
  31.                         context.write(mKey, mValue);
  32.                 }
  33.         }

  35.         public static class MyWordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
  36.                 Text rkey = new Text();
  37.                 IntWritable rvalue = new IntWritable(1);
  38.                 @Override
  39.                 protected void reduce(Text key, Iterable<IntWritable> values,Context context)
  40.                                 throws IOException, InterruptedException {

  42.                         int n=0;
  43.                         for(IntWritable value :values){
  44.                                 n+= value.get();
  45.                         }
  46.                         rvalue.set(n);
  47.                         context.write(key, rvalue);
  48.                 }
  49.         }

  51.         @Override
  52.         public int run(String[] args) throws Exception {
  53.                 //valid the parameters
  54.                 if(args.length !=2){
  55.                         return -1;
  56.                 }

  58.                 Job job = Job.getInstance(getConf(), "MyWordCountJob");
  59.                 job.setJarByClass(MyWordCountJob.class);

  61.                 Path inPath = new Path(args[0]);
  62.                 Path outPath = new Path(args[1]);

  64.                 outPath.getFileSystem(getConf()).delete(outPath,true);
  65.                 TextInputFormat.setInputPaths(job, inPath);
  66.                 TextOutputFormat.setOutputPath(job, outPath);


  69.                 job.setMapperClass(MyWordCountJob.MyWordCountMapper.class);
  70.                 job.setReducerClass(MyWordCountJob.MyWordCountReducer.class);

  72.                 job.setInputFormatClass(MyCombinedFilesInputFormat.class);
  73.                 MyCombinedFilesInputFormat.setMaxInputSplitSize(job, 1024*1024*64);
  74.                 job.setOutputFormatClass(TextOutputFormat.class);
  75.                 job.setMapOutputKeyClass(Text.class);
  76.                 job.setMapOutputValueClass(IntWritable.class);
  77.                 job.setOutputKeyClass(Text.class);
  78.                 job.setOutputValueClass(IntWritable.class);


  81.                 return job.waitForCompletion(true)?0:1;
  82.         }
  83.         public static void main(String [] args){
  84.                 int result = 0;
  85.                 try {
  86.                         result = ToolRunner.run(new Configuration(), new MyWordCountJob(), args);
  87.                 } catch (Exception e) {
  88.                         e.printStackTrace();
  89.                 }
  90.                 System.exit(result);
  91.         }

  93. }

而且使用自定義的MyRecordReader的好處不限於此，稍後一篇討論使用TotalOrderPartitioner時會發現，對於詞頻統計，使用TotalOrderPartitioner 自定義RecordReader是必要的。

來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/30066956/viewspace-2109264/，如需轉載，請註明出處，否則將追究法律責任。

相關文章